From bd8124dcbfef230aef7a38f2fbbe18edcb2d5285 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 16:05:04 -0500 Subject: [PATCH 1/8] Move non-hook function out of hooks submodule --- tox_poetry_installer/hooks.py | 39 +------------------------------ tox_poetry_installer/utilities.py | 37 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index b95a312..c0d8cef 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -8,7 +8,6 @@ from typing import List from typing import Optional from poetry.core.packages import Package as PoetryPackage -from poetry.poetry import Poetry from tox import hookimpl from tox import reporter from tox.action import Action as ToxAction @@ -104,7 +103,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional ) if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist: - project_deps: List[PoetryPackage] = _find_project_dependencies( + project_deps: List[PoetryPackage] = utilities.find_project_dependencies( venv, poetry, package_map ) else: @@ -127,39 +126,3 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional utilities.install_to_venv(poetry, venv, dependencies) return venv.envconfig.require_locked_deps or None - - -def _find_project_dependencies( - venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap -) -> List[PoetryPackage]: - """Install the dependencies of the project package - - Install all primary dependencies of the project package. - - :param venv: Tox virtual environment to install the packages to - :param poetry: Poetry object the packages were sourced from - :param packages: Mapping of package names to the corresponding package object - """ - - base_dependencies: List[PoetryPackage] = [ - packages[item.name] - for item in poetry.package.requires - if not item.is_optional() - ] - - extra_dependencies: List[PoetryPackage] = [] - for extra in venv.envconfig.extras: - try: - extra_dependencies += [ - packages[item.name] for item in poetry.package.extras[extra] - ] - except KeyError: - raise exceptions.ExtraNotFoundError( - f"Environment '{venv.name}' specifies project extra '{extra}' which was not found in the lockfile" - ) from None - - dependencies: List[PoetryPackage] = [] - for dep in base_dependencies + extra_dependencies: - dependencies += utilities.find_transients(packages, dep.name.lower()) - - return dependencies diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 5c44910..1aee525 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -1,6 +1,7 @@ """Helper utility functions, usually bridging Tox and Poetry functionality""" import sys from pathlib import Path +from typing import List from typing import Sequence from typing import Set @@ -135,3 +136,39 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry: raise exceptions.SkipEnvironment( "Project does not use Poetry for env management, skipping installation of locked dependencies" ) from None + + +def find_project_dependencies( + venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap +) -> List[PoetryPackage]: + """Install the dependencies of the project package + + Install all primary dependencies of the project package. + + :param venv: Tox virtual environment to install the packages to + :param poetry: Poetry object the packages were sourced from + :param packages: Mapping of package names to the corresponding package object + """ + + base_dependencies: List[PoetryPackage] = [ + packages[item.name] + for item in poetry.package.requires + if not item.is_optional() + ] + + extra_dependencies: List[PoetryPackage] = [] + for extra in venv.envconfig.extras: + try: + extra_dependencies += [ + packages[item.name] for item in poetry.package.extras[extra] + ] + except KeyError: + raise exceptions.ExtraNotFoundError( + f"Environment '{venv.name}' specifies project extra '{extra}' which was not found in the lockfile" + ) from None + + dependencies: List[PoetryPackage] = [] + for dep in base_dependencies + extra_dependencies: + dependencies += find_transients(packages, dep.name.lower()) + + return dependencies From 01635c50c797f394edb30d75368a1d50987fffe9 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 17:11:32 -0500 Subject: [PATCH 2/8] Update logging integration to improve standardization Standardize language in logging messages Move system version to constants Fixes #3 --- tox_poetry_installer/constants.py | 14 +++++++- tox_poetry_installer/hooks.py | 56 +++++++++++++++++-------------- tox_poetry_installer/utilities.py | 10 ++---- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/tox_poetry_installer/constants.py b/tox_poetry_installer/constants.py index a99f1b5..2e7531e 100644 --- a/tox_poetry_installer/constants.py +++ b/tox_poetry_installer/constants.py @@ -5,8 +5,11 @@ in this module. All constants should be type hinted. """ +import sys from typing import Tuple +from poetry.core.semver.version import Version + from tox_poetry_installer import __about__ @@ -16,4 +19,13 @@ PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<") # Prefix all reporter messages should include to indicate that they came from this module in the # console output. -REPORTER_PREFIX = f"[{__about__.__title__}]:" +REPORTER_PREFIX: str = f"[{__about__.__title__}]:" + + +# Semver compatible version of the current python platform version. Used for checking +# whether a package is compatible with the current python system version +PLATFORM_VERSION: Version = Version( + major=sys.version_info.major, + minor=sys.version_info.minor, + patch=sys.version_info.micro, +) diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index c0d8cef..6e30dfa 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -14,6 +14,7 @@ from tox.action import Action as ToxAction from tox.config import Parser as ToxParser from tox.venv import VirtualEnv as ToxVirtualEnv +from tox_poetry_installer import __about__ from tox_poetry_installer import constants from tox_poetry_installer import exceptions from tox_poetry_installer import utilities @@ -71,30 +72,30 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional f"{constants.REPORTER_PREFIX} Loaded project pyproject.toml from {poetry.file}" ) - if venv.envconfig.require_locked_deps and venv.envconfig.deps: - raise exceptions.LockedDepsRequiredError( - f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies" + try: + if venv.envconfig.require_locked_deps and venv.envconfig.deps: + raise exceptions.LockedDepsRequiredError( + f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies" + ) + + package_map: PackageMap = { + package.name: package + for package in poetry.locker.locked_repository(True).packages + } + + if venv.envconfig.install_dev_deps: + dev_deps: List[PoetryPackage] = [ + dep + for dep in package_map.values() + if dep not in poetry.locker.locked_repository(False).packages + ] + else: + dev_deps = [] + + reporter.verbosity1( + f"{constants.REPORTER_PREFIX} Identified {len(dev_deps)} development dependencies to install to env" ) - package_map: PackageMap = { - package.name: package - for package in poetry.locker.locked_repository(True).packages - } - - if venv.envconfig.install_dev_deps: - dev_deps: List[PoetryPackage] = [ - dep - for dep in package_map.values() - if dep not in poetry.locker.locked_repository(False).packages - ] - else: - dev_deps = [] - - reporter.verbosity1( - f"{constants.REPORTER_PREFIX} Identified {len(dev_deps)} development dependencies to install to env" - ) - - try: env_deps: List[PoetryPackage] = [] for dep in venv.envconfig.locked_deps: env_deps += utilities.find_transients(package_map, dep.lower()) @@ -115,13 +116,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional f"{constants.REPORTER_PREFIX} Identified {len(project_deps)} project dependencies to install to env" ) except exceptions.ToxPoetryInstallerException as err: - venv.status = "lockfile installation failed" + venv.status = err.__class__.__name__ reporter.error(f"{constants.REPORTER_PREFIX} {err}") + return False + except Exception as err: + venv.status = "InternalError" + reporter.error(f"{constants.REPORTER_PREFIX} Internal plugin error: {err}") raise err dependencies = list(set(dev_deps + env_deps + project_deps)) - reporter.verbosity0( - f"{constants.REPORTER_PREFIX} Installing {len(dependencies)} dependencies to env '{action.name}'" + action.setactivity( + __about__.__title__, + f"Installing {len(dependencies)} dependencies from Poetry lock file", ) utilities.install_to_venv(poetry, venv, dependencies) diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 1aee525..55659bc 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -6,7 +6,6 @@ from typing import Sequence from typing import Set from poetry.core.packages import Package as PoetryPackage -from poetry.core.semver.version import Version from poetry.factory import Factory as PoetryFactory from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller from poetry.io.null_io import NullIO as PoetryNullIO @@ -64,11 +63,6 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac def find_deps_of_deps(name: str, searched: Set[str]) -> PackageMap: package = packages[name] - local_version = Version( - major=sys.version_info.major, - minor=sys.version_info.minor, - patch=sys.version_info.micro, - ) transients: PackageMap = {} searched.update([name]) @@ -79,9 +73,9 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac reporter.verbosity2( f"{constants.REPORTER_PREFIX} Skip {package}: designated unsafe by Poetry" ) - elif not package.python_constraint.allows(local_version): + elif not package.python_constraint.allows(constants.PLATFORM_VERSION): reporter.verbosity2( - f"{constants.REPORTER_PREFIX} Skip {package}: incompatible Python requirement '{package.python_constraint}' for current version '{local_version}'" + f"{constants.REPORTER_PREFIX} Skip {package}: incompatible Python requirement '{package.python_constraint}' for current version '{constants.PLATFORM_VERSION}'" ) elif package.platform is not None and package.platform != sys.platform: reporter.verbosity2( From afad7663f026e0d80c68f3034dffa996f3910c58 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 17:15:00 -0500 Subject: [PATCH 3/8] Fix clean target not deleting pycache --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3bf0260..1cfcf27 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ clean-py: rm --recursive --force ./dist rm --recursive --force ./build rm --recursive --force ./*.egg-info - rm --recursive --force __pycache__/ + rm --recursive --force ./**/__pycache__/ clean: clean-tox clean-py; ## Clean temp build/cache files and directories From 872f6b0892a0be7597eaa9814cde003c02554ebf Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 17:50:29 -0500 Subject: [PATCH 4/8] Update poetry dependency to be installed as optional extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When running 'poetry remove tox-poetry-installer' in the same env as poetry is installed to, poetry will uninstall itself. This is, obviously, not ideal. This change makes poetry an optional dependency so that the plugin can be installed (and uninstalled) alongside poetry in the same env without breaking the poetry installation. The intention is that the plugin can be installed with the 'poetry' extra when being installed to an isolated environment where poetry is not otherwise available. This is a mitigation of Issue #2 as an alternative to vendorization of the enitre poetry project 😬 --- poetry.lock | 66 +++++++++++++++++++++++++------------------------- pyproject.toml | 7 ++++-- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4969431..6f68ed7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -123,7 +123,7 @@ name = "cachecontrol" version = "0.12.6" description = "httplib2 caching for requests" category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] @@ -148,7 +148,7 @@ name = "cachy" version = "0.3.0" description = "Cachy provides a simple yet effective caching library." category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] @@ -169,7 +169,7 @@ name = "cffi" version = "1.14.4" description = "Foreign Function Interface for Python calling C code." category = "main" -optional = false +optional = true python-versions = "*" [package.dependencies] @@ -196,7 +196,7 @@ name = "cleo" version = "0.8.1" description = "Cleo allows you to create beautiful and testable command-line interfaces." category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] @@ -215,7 +215,7 @@ name = "clikit" version = "0.6.2" description = "CliKit is a group of utilities to build beautiful and testable command line interfaces." category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] @@ -247,7 +247,7 @@ name = "crashtest" version = "0.3.1" description = "Manage Python errors with ease" category = "main" -optional = false +optional = true python-versions = ">=3.6,<4.0" [[package]] @@ -255,7 +255,7 @@ name = "cryptography" version = "3.2.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" -optional = false +optional = true python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [package.dependencies] @@ -263,11 +263,11 @@ cffi = ">=1.8,<1.11.3 || >1.11.3" six = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] +test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] name = "dataclasses" @@ -344,7 +344,7 @@ name = "html5lib" version = "1.1" description = "HTML parser based on the WHATWG HTML specification" category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] @@ -477,7 +477,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" parso = ">=0.7.0,<0.8.0" [package.extras] -qa = ["flake8 (3.7.9)"] +qa = ["flake8 (==3.7.9)"] testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] [[package]] @@ -485,7 +485,7 @@ name = "jeepney" version = "0.6.0" description = "Low-level, pure Python DBus protocol wrapper." category = "main" -optional = false +optional = true python-versions = ">=3.6" [package.extras] @@ -496,7 +496,7 @@ name = "keyring" version = "21.5.0" description = "Store and access your passwords safely." category = "main" -optional = false +optional = true python-versions = ">=3.6" [package.dependencies] @@ -507,7 +507,7 @@ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "pytest-black (>=0.3.7)", "pytest-mypy"] [[package]] name = "lazy-object-proxy" @@ -522,7 +522,7 @@ name = "lockfile" version = "0.12.2" description = "Platform-independent file locking module" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -538,7 +538,7 @@ name = "msgpack" version = "1.0.0" description = "MessagePack (de)serializer." category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -601,7 +601,7 @@ name = "pastel" version = "0.2.1" description = "Bring colors to your terminal." category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] @@ -644,7 +644,7 @@ name = "pkginfo" version = "1.6.1" description = "Query metadatdata from sdists / bdists / installed packages." category = "main" -optional = false +optional = true python-versions = "*" [package.extras] @@ -669,7 +669,7 @@ name = "poetry" version = "1.1.4" description = "Python dependency management and packaging made easy." category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] @@ -764,7 +764,7 @@ name = "pycparser" version = "2.20" description = "C parser in Python" category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] @@ -780,7 +780,7 @@ name = "pylev" version = "1.3.0" description = "A pure Python Levenshtein implementation that's not freaking GPL'd." category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -826,7 +826,7 @@ py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (0.780)"] +checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -842,14 +842,14 @@ coverage = ">=4.4" pytest = ">=4.6" [package.extras] -testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] [[package]] name = "pywin32-ctypes" version = "0.2.0" description = "" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -895,14 +895,14 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "requests-toolbelt" version = "0.9.1" description = "A utility belt for advanced users of python-requests" category = "main" -optional = false +optional = true python-versions = "*" [package.dependencies] @@ -950,7 +950,7 @@ name = "secretstorage" version = "3.3.0" description = "Python bindings to FreeDesktop.org Secret Service API" category = "main" -optional = false +optional = true python-versions = ">=3.6" [package.dependencies] @@ -962,7 +962,7 @@ name = "shellingham" version = "1.3.2" description = "Tool to Detect Surrounding Shell" category = "main" -optional = false +optional = true python-versions = "!=3.0,!=3.1,!=3.2,!=3.3,>=2.6" [[package]] @@ -1006,7 +1006,7 @@ name = "tomlkit" version = "0.7.0" description = "Style preserving TOML library" category = "main" -optional = false +optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] @@ -1073,7 +1073,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] -socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "virtualenv" @@ -1108,7 +1108,7 @@ name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" category = "main" -optional = false +optional = true python-versions = "*" [[package]] @@ -1129,12 +1129,12 @@ python-versions = ">=3.6" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "8eea42cb6c60df03376bb264b444ccd0a63a211122edc6625284d57204295273" +content-hash = "b466984a069123fa9c5af59c37a6a8c8404742eb90a45c37fc6d73d87e6383f8" [metadata.files] appdirs = [ diff --git a/pyproject.toml b/pyproject.toml index 34ee599..6efee11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,11 +32,14 @@ classifiers = [ [tool.poetry.plugins.tox] poetry_installer = "tox_poetry_installer" +[tool.poetry.extras] +poetry = ["poetry"] + [tool.poetry.dependencies] python = "^3.6.1" -poetry = "^1.0.0" +poetry = {version = "^1.0.0", optional = true} poetry-core = "^1.0.0" -tox = "^2.3.0 || ^3.0.0" +tox = "^3.0.0" [tool.poetry.dev-dependencies] bandit = "^1.6.2" From 604e60d5674d41f4e6dfd10cc3ca3faf454f7693 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 17:56:51 -0500 Subject: [PATCH 5/8] Bump feature version --- pyproject.toml | 2 +- tox_poetry_installer/__about__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6efee11..b4ad101 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tox-poetry-installer" -version = "0.5.2" +version = "0.6.0" license = "MIT" authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"] description = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile" diff --git a/tox_poetry_installer/__about__.py b/tox_poetry_installer/__about__.py index 4b21a84..af40cd6 100644 --- a/tox_poetry_installer/__about__.py +++ b/tox_poetry_installer/__about__.py @@ -1,7 +1,7 @@ # pylint: disable=missing-docstring __title__ = "tox-poetry-installer" __summary__ = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile" -__version__ = "0.5.2" +__version__ = "0.6.0" __url__ = "https://github.com/enpaul/tox-poetry-installer/" __license__ = "MIT" __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"] From 5a23c05f17be1396f52422dd9548cd8e2ea8b86a Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 19:25:40 -0500 Subject: [PATCH 6/8] Add internally proxied poetry module to support runtime in non-Poetry envs Add the _poetry submodule to support importing Poetry internals at runtime rather than import time. This allows the plugin to be run, and either skipped or errored, without crashing tox --- poetry.lock | 42 ++++++++++++++++-------------- tox_poetry_installer/_poetry.py | 41 +++++++++++++++++++++++++++++ tox_poetry_installer/exceptions.py | 5 ++++ tox_poetry_installer/utilities.py | 33 +++++++++++++---------- 4 files changed, 88 insertions(+), 33 deletions(-) create mode 100644 tox_poetry_installer/_poetry.py diff --git a/poetry.lock b/poetry.lock index 6f68ed7..64a7248 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8,8 +8,8 @@ python-versions = "*" [[package]] name = "appnope" -version = "0.1.0" -description = "Disable App Nap on OS X 10.9" +version = "0.1.2" +description = "Disable App Nap on macOS >= 10.9" category = "dev" optional = false python-versions = "*" @@ -575,7 +575,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.4" +version = "20.7" description = "Core utilities for Python packages" category = "main" optional = false @@ -583,7 +583,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" -six = "*" [[package]] name = "parso" @@ -983,7 +982,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "stevedore" -version = "3.2.2" +version = "3.3.0" description = "Manage dynamic plugins for Python applications" category = "dev" optional = false @@ -1131,10 +1130,13 @@ python-versions = ">=3.6" docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +[extras] +poetry = ["poetry"] + [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "b466984a069123fa9c5af59c37a6a8c8404742eb90a45c37fc6d73d87e6383f8" +content-hash = "a5ba6181fc3728d85a60b2e089b9afe2d5bf75f361526e6972d48a42e5075c32" [metadata.files] appdirs = [ @@ -1142,8 +1144,8 @@ appdirs = [ {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] appnope = [ - {file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"}, - {file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"}, + {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, + {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, ] "aspy.refactor-imports" = [ {file = "aspy.refactor_imports-2.1.1-py2.py3-none-any.whl", hash = "sha256:9df76bf19ef81620068b785a386740ab3c8939fcbdcebf20c4a4e0057230d782"}, @@ -1468,8 +1470,8 @@ nodeenv = [ {file = "nodeenv-1.5.0.tar.gz", hash = "sha256:ab45090ae383b716c4ef89e690c41ff8c2b257b85b309f01f3654df3d084bd7c"}, ] packaging = [ - {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, - {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, + {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, + {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, ] parso = [ {file = "parso-0.7.1-py2.py3-none-any.whl", hash = "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea"}, @@ -1643,29 +1645,22 @@ requests-toolbelt = [ {file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"}, {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"}, {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"}, {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"}, {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"}, {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"}, {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"}, {file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"}, ] safety = [ @@ -1689,8 +1684,8 @@ smmap = [ {file = "smmap-3.0.4.tar.gz", hash = "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24"}, ] stevedore = [ - {file = "stevedore-3.2.2-py3-none-any.whl", hash = "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62"}, - {file = "stevedore-3.2.2.tar.gz", hash = "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0"}, + {file = "stevedore-3.3.0-py3-none-any.whl", hash = "sha256:50d7b78fbaf0d04cd62411188fa7eedcb03eb7f4c4b37005615ceebe582aa82a"}, + {file = "stevedore-3.3.0.tar.gz", hash = "sha256:3a5bbd0652bf552748871eaa73a4a8dc2899786bc497a2aa1fcb4dcdb0debeee"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, @@ -1716,19 +1711,28 @@ typed-ast = [ {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f"}, {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298"}, {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d"}, {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, + {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d"}, + {file = "typed_ast-1.4.1-cp39-cp39-win32.whl", hash = "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395"}, + {file = "typed_ast-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ diff --git a/tox_poetry_installer/_poetry.py b/tox_poetry_installer/_poetry.py new file mode 100644 index 0000000..357dfec --- /dev/null +++ b/tox_poetry_installer/_poetry.py @@ -0,0 +1,41 @@ +"""You've heard of vendoirization, now get ready for internal namespace shadowing + +Poetry is an optional dependency of this package explicitly to support the use case of having the +plugin and the `poetry` package installed to the same python environment; this is most common in +containers and/or CI. In this case there are two potential problems that can arise in this case: + +* The installation of the plugin overwrites the installed version of Poetry resulting in + compatibility issues. +* Running `poetry install --no-dev`, when this plugin is in the dev-deps, results in poetry being + uninstalled from the environment. + +To support these edge cases, and more broadly to support not messing with a system package manager, +the `poetry` package dependency is listed as optional dependency. This allows the plugin to be +installed to the same environment as Poetry and import that same Poetry installation here. + +However, simply importing Poetry on the assumption that it is installed breaks another valid use +case: having this plugin installed alongside Tox when not using a Poetry-based project. To account +for this the imports in this module are isolated and the resultant import error that would result +is converted to an internal error that can be caught by callers. Rather than importing this module +at the module scope it is imported into function scope wherever Poetry components are needed. This +moves import errors from load time to runtime which allows the plugin to be skipped if Poetry isn't +installed and/or a more helpful error be raised within the Tox framework. +""" +# pylint: disable=unused-import +import sys + +from tox_poetry_installer import exceptions + + +try: + from poetry.factory import Factory + from poetry.installation.pip_installer import PipInstaller + from poetry.io.null_io import NullIO + from poetry.poetry import Poetry + from poetry.puzzle.provider import Provider + from poetry.utils.env import VirtualEnv +except ImportError: + raise exceptions.PoetryNotInstalledError( + f"No version of Poetry could be imported under the current environment for '{sys.executable}'", + sys.path, + ) from None diff --git a/tox_poetry_installer/exceptions.py b/tox_poetry_installer/exceptions.py index d11facd..44d476f 100644 --- a/tox_poetry_installer/exceptions.py +++ b/tox_poetry_installer/exceptions.py @@ -6,6 +6,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst ToxPoetryInstallerException +-- SkipEnvironment + | +-- PoetryNotInstalledError +-- LockedDepVersionConflictError +-- LockedDepNotFoundError +-- ExtraNotFoundError @@ -22,6 +23,10 @@ class SkipEnvironment(ToxPoetryInstallerException): """Current environment does not meet preconditions and should be skipped by the plugin""" +class PoetryNotInstalledError(SkipEnvironment): + """No version of Poetry could be imported from the current Python environment""" + + class LockedDepVersionConflictError(ToxPoetryInstallerException): """Locked dependencies cannot specify an alternate version for installation""" diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 55659bc..838d924 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -1,17 +1,15 @@ """Helper utility functions, usually bridging Tox and Poetry functionality""" +# Silence this one globally to support the internal function imports for the proxied poetry module. +# See the docstring in 'tox_poetry_installer._poetry' for more context. +# pylint: disable=import-outside-toplevel import sys +import typing from pathlib import Path from typing import List from typing import Sequence from typing import Set from poetry.core.packages import Package as PoetryPackage -from poetry.factory import Factory as PoetryFactory -from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller -from poetry.io.null_io import NullIO as PoetryNullIO -from poetry.poetry import Poetry -from poetry.puzzle.provider import Provider as PoetryProvider -from poetry.utils.env import VirtualEnv as PoetryVirtualEnv from tox import reporter from tox.action import Action as ToxAction from tox.venv import VirtualEnv as ToxVirtualEnv @@ -20,9 +18,12 @@ from tox_poetry_installer import constants from tox_poetry_installer import exceptions from tox_poetry_installer.datatypes import PackageMap +if typing.TYPE_CHECKING: + from tox_poetry_installer import _poetry + def install_to_venv( - poetry: Poetry, venv: ToxVirtualEnv, packages: Sequence[PoetryPackage] + poetry: "_poetry.Poetry", venv: ToxVirtualEnv, packages: Sequence[PoetryPackage] ): """Install a bunch of packages to a virtualenv @@ -30,14 +31,15 @@ def install_to_venv( :param venv: Tox virtual environment to install the packages to :param packages: List of packages to install to the virtual environment """ + from tox_poetry_installer import _poetry reporter.verbosity1( f"{constants.REPORTER_PREFIX} Installing {len(packages)} packages to environment at {venv.envconfig.envdir}" ) - installer = PoetryPipInstaller( - env=PoetryVirtualEnv(path=Path(venv.envconfig.envdir)), - io=PoetryNullIO(), + installer = _poetry.PipInstaller( + env=_poetry.VirtualEnv(path=Path(venv.envconfig.envdir)), + io=_poetry.NullIO(), pool=poetry.pool, ) @@ -58,6 +60,7 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac .. note:: The package corresponding to the dependency named by ``dependency_name`` is included in the list of returned packages. """ + from tox_poetry_installer import _poetry try: @@ -66,7 +69,7 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac transients: PackageMap = {} searched.update([name]) - if name in PoetryProvider.UNSAFE_PACKAGES: + if name in _poetry.Provider.UNSAFE_PACKAGES: reporter.warning( f"{constants.REPORTER_PREFIX} Installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'" ) @@ -109,8 +112,10 @@ def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPac ) from None -def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry: +def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poetry": """Check that the local project environment meets expectations""" + from tox_poetry_installer import _poetry + # Skip running the plugin for the packaging environment. PEP-517 front ends can handle # that better than we can, so let them do their thing. More to the point: if you're having # problems in the packaging env that this plugin would solve, god help you. @@ -120,7 +125,7 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry: ) try: - return PoetryFactory().create_poetry(venv.envconfig.config.toxinidir) + return _poetry.Factory().create_poetry(venv.envconfig.config.toxinidir) # Support running the plugin when the current tox project does not use Poetry for its # environment/dependency management. # @@ -133,7 +138,7 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> Poetry: def find_project_dependencies( - venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap + venv: ToxVirtualEnv, poetry: "_poetry.Poetry", packages: PackageMap ) -> List[PoetryPackage]: """Install the dependencies of the project package From db761d49c1f0526fd9e0eb21831285c4b219fca0 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Fri, 4 Dec 2020 22:28:27 -0500 Subject: [PATCH 7/8] Add poetry extra install to tox config to support CI --- tox.ini | 2 ++ tox_poetry_installer/utilities.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index bf4ce7c..cbee73c 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ skip_missing_interpreters = true [testenv] description = Run the tests require_locked_deps = true +extras = + poetry locked_deps = pytest pytest-cov diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 838d924..09e6439 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -157,6 +157,9 @@ def find_project_dependencies( extra_dependencies: List[PoetryPackage] = [] for extra in venv.envconfig.extras: + reporter.verbosity1( + f"{constants.REPORTER_PREFIX} Processing project extra '{extra}'" + ) try: extra_dependencies += [ packages[item.name] for item in poetry.package.extras[extra] From 52aaeba93c4ed16dc06a649872db4a459862c1c7 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Sat, 5 Dec 2020 12:36:35 -0500 Subject: [PATCH 8/8] Overhaul CI to improve resilance and efficiency Add caching for pip and poetry downloads to reduce runtime Add pinned pip version Add poetry installation of local project Remove bare pip install for local project installation --- .github/scripts/setup-env.sh | 72 ++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yaml | 51 +++++++++++++++++++------ 2 files changed, 111 insertions(+), 12 deletions(-) create mode 100755 .github/scripts/setup-env.sh diff --git a/.github/scripts/setup-env.sh b/.github/scripts/setup-env.sh new file mode 100755 index 0000000..1b43d94 --- /dev/null +++ b/.github/scripts/setup-env.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Environment setup script for the local project. Intended to be used with automation +# to create a repeatable local environment for tests to be run in. The python env +# this script creates can be accessed at the location defined by the CI_VENV variable +# below. + +set -e; + +# ##### Prereqs ##### +# +# Set global vars for usage in the script, create the cache directory so we can rely +# on that existing, then dump some diagnostic info for later reference. +# +CI_VENV=$HOME/ci; +CI_CACHE=$HOME/.cache; +CI_CACHE_GET_POETRY="$CI_CACHE/get-poetry.py"; +CI_POETRY=$HOME/.poetry/bin/poetry; +CI_VENV_PIP="$CI_VENV/bin/pip"; +CI_VENV_PIP_VERSION=19.3.1; +CI_VENV_TOX="$CI_VENV/bin/tox"; + +mkdir --parents "$CI_CACHE"; + +command -v python; +python --version; + +# ##### Install Poetry ##### +# +# Download the poetry install script to the cache directory and then install poetry. +# After dump the poetry version for later reference. +# +curl https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py \ + --output "$CI_CACHE_GET_POETRY" \ + --silent \ + --show-error \ + --location; +python "$CI_CACHE_GET_POETRY" --yes 1>/dev/null; + +python "$CI_POETRY" --version --no-ansi; + +# ##### Setup Runtime Venv ##### +# +# Create a virtual environment for poetry to use, upgrade pip in that venv to a pinned +# version, then install the current project to the venv. +# +# Note 1: Poetry, Tox, and this project plugin all use pip under the hood for package +# installation. This means that even though we are creating up to eight venvs +# during a given CI run they all share the same download cache. +# Note 2: The "VIRTUAL_ENV=$CI_VENV" prefix on the poetry commands below sets the venv +# that poetry will use for operations. There is no CLI flag for poetry that +# directs it to use a given environment, but if it finds itself in an existing +# environment it will use it and skip environment creation. +# +python -m venv "$CI_VENV"; + +$CI_VENV_PIP install "pip==$CI_VENV_PIP_VERSION" \ + --upgrade \ + --quiet; + +VIRTUAL_ENV=$CI_VENV "$CI_POETRY" install \ + --extras poetry \ + --quiet \ + --no-ansi \ + &>/dev/null; + +# ##### Print Debug Info ##### +# +# Print the pip and tox versions (which will include registered plugins) +# +$CI_VENV_PIP --version; +echo "tox $($CI_VENV_TOX --version)"; diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6fa8c21..30c31b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,24 +20,51 @@ jobs: - version: 3.9 toxenv: py39 steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python.version }} + - name: Checkout + uses: actions/checkout@v2 + - name: Setup:python${{ matrix.python.version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python.version }} - - name: Install project - run: pip install . - - name: Run tests via ${{ matrix.python.toxenv }} - run: tox -e ${{ matrix.python.toxenv }} + - name: Setup:cache + uses: actions/cache@v2 + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/cache + ~/.poetry + # Including the hashed poetry.lock in the cache slug ensures that the cache + # will be invalidated, and thus all packages will be redownloaded, if the + # lockfile is updated + key: ${{ runner.os }}-${{ matrix.python.toxenv }}-${{ hashFiles('**/poetry.lock') }} + - name: Setup:env + run: .github/scripts/setup-env.sh + - name: Run:${{ matrix.python.toxenv }} + run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} Check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Checkout + uses: actions/checkout@v2 + - name: Setup:python3.8 uses: actions/setup-python@v1 with: python-version: 3.8 - - name: Install project - run: pip install . - - name: Run meta checks - run: tox -e static -e static-tests -e security + - name: Setup:cache + uses: actions/cache@v2 + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/cache + ~/.poetry + # Hardcoded 'py38' slug here lets this cache piggyback on the 'py38' cache + # that is generated for the tests above + key: ${{ runner.os }}-py38-${{ hashFiles('**/poetry.lock') }} + - name: Setup:env + run: .github/scripts/setup-env.sh + - name: Run:static + run: $HOME/ci/bin/tox -e static + - name: Run:static-tests + run: $HOME/ci/bin/tox -e static-tests + - name: Run:security + run: $HOME/ci/bin/tox -e security