From dbbbf8186fdad7fdd3d37bb4afb7be2be07fc832 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 16:18:41 -0400 Subject: [PATCH 1/8] Update default setup to use parallelized dep installation Move deprecation warnings to the precondition function Deprecate --parallelize-locked-install option Add --parallel-install-threads option --- tox_poetry_installer/constants.py | 3 +++ tox_poetry_installer/hooks.py | 39 +++++++++++++++++++------------ tox_poetry_installer/installer.py | 5 ++-- tox_poetry_installer/utilities.py | 13 +++++++++++ 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/tox_poetry_installer/constants.py b/tox_poetry_installer/constants.py index 3be4fd8..ccfc6f3 100644 --- a/tox_poetry_installer/constants.py +++ b/tox_poetry_installer/constants.py @@ -21,3 +21,6 @@ REPORTER_PREFIX: str = f"{__about__.__title__}:" # Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"} + +# Number of threads to use for installing dependencies by default +DEFAULT_INSTALL_THREADS: int = 10 diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index 7c880ee..871c9f6 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -12,6 +12,7 @@ from tox.config import Parser as ToxParser from tox.venv import VirtualEnv as ToxVirtualEnv from tox_poetry_installer import __about__ +from tox_poetry_installer import constants from tox_poetry_installer import exceptions from tox_poetry_installer import installer from tox_poetry_installer import logger @@ -31,7 +32,7 @@ def tox_addoption(parser: ToxParser): "--require-poetry", action="store_true", dest="require_poetry", - help="Trigger a failure if Poetry is not available to Tox", + help="(deprecated) Trigger a failure if Poetry is not available to Tox", ) parser.add_argument( @@ -39,7 +40,15 @@ def tox_addoption(parser: ToxParser): type=int, dest="parallelize_locked_install", default=None, - help="Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel", + help="(deprecated) Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel", + ) + + parser.add_argument( + "--parallel-install-threads", + type=int, + dest="parallel_install_threads", + default=constants.DEFAULT_INSTALL_THREADS, + help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation", ) parser.add_testenv_attribute( @@ -82,13 +91,6 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional :param action: Tox action object """ - if venv.envconfig.config.option.require_poetry: - logger.warning( - "DEPRECATION WARNING: The '--require-poetry' runtime option is deprecated and will be " - "removed in version 1.0.0. Please update test environments that require Poetry to " - "set the 'require_poetry = true' option in tox.ini" - ) - try: poetry = utilities.check_preconditions(venv, action) except exceptions.SkipEnvironment as err: @@ -158,11 +160,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional raise err dependencies = dev_deps + env_deps + project_deps - log_parallel = ( - f" (using {venv.envconfig.config.option.parallelize_locked_install} threads)" - if venv.envconfig.config.option.parallelize_locked_install - else "" - ) + if ( + venv.envconfig.config.option.parallel_install_threads + != constants.DEFAULT_INSTALL_THREADS + ): + parallel_threads = venv.envconfig.config.option.parallel_install_threads + else: + parallel_threads = ( + venv.envconfig.config.option.parallelize_locked_install + if venv.envconfig.config.option.parallelize_locked_install is not None + else constants.DEFAULT_INSTALL_THREADS + ) + log_parallel = f" (using {parallel_threads} threads)" if parallel_threads else "" action.setactivity( __about__.__title__, @@ -172,7 +181,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional poetry, venv, dependencies, - venv.envconfig.config.option.parallelize_locked_install, + parallel_threads, ) return venv.envconfig.require_locked_deps or None diff --git a/tox_poetry_installer/installer.py b/tox_poetry_installer/installer.py index 1685ea0..bbde07f 100644 --- a/tox_poetry_installer/installer.py +++ b/tox_poetry_installer/installer.py @@ -5,7 +5,6 @@ import concurrent.futures import contextlib import typing -from typing import Optional from typing import Sequence from typing import Set @@ -23,7 +22,7 @@ def install( poetry: "_poetry.Poetry", venv: ToxVirtualEnv, packages: Sequence[PoetryPackage], - parallels: Optional[int] = None, + parallels: int = 0, ): """Install a bunch of packages to a virtualenv @@ -55,7 +54,7 @@ def install( enables/disables the usage of the parallel thread pooler depending on the value of the ``parallels`` parameter. """ - if parallels: + if parallels > 0: with concurrent.futures.ThreadPoolExecutor( max_workers=parallels ) as executor: diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 6f7ac80..87e1d98 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -43,6 +43,19 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poet f"Skipping isolated packaging build env '{action.name}'" ) + if venv.envconfig.config.option.require_poetry: + logger.warning( + "DEPRECATION: The '--require-poetry' runtime option is deprecated and will be " + "removed in version 1.0.0. Please update test environments that require Poetry to " + "set the 'require_poetry = true' option in tox.ini" + ) + + if venv.envconfig.config.option.parallelize_locked_install is not None: + logger.warning( + "DEPRECATION: The '--parallelize-locked-install' option is deprecated and will " + "be removed in version 1.0.0. Please use the '--parallel-install-threads' option." + ) + from tox_poetry_installer import _poetry try: From e4139d9875c714092c1c8d5ad1485e662874b847 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 16:26:36 -0400 Subject: [PATCH 2/8] Update docs with new runtime option Fix formatting in example path --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 49891a4..9f5ba65 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ adding the below to `tox.ini`, though this is also not recommended: ```ini requires = - tox-poetry-installer[poetry] == 0.7.0 + tox-poetry-installer[poetry] == 0.8.0 ``` After installing, check that Tox recognizes the plugin by running @@ -89,7 +89,7 @@ After installing, check that Tox recognizes the plugin by running ``` 3.20.0 imported from .venv/lib64/python3.8/site-packages/tox/__init__.py registered plugins: - tox-poetry-installer-0.7.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer.py + tox-poetry-installer-0.8.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer/__init__.py ``` ### Quick Start @@ -206,14 +206,17 @@ configuration section. All arguments listed below can be passed to the `tox` command to modify runtime behavior of the plugin. -| Argument | Type | Default | Description | -| :----------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--parallelize-locked-install` | Integer | `0` | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. The default, `0`, disables the parallel install so that dependencies are installed sequentially. | +| Argument | Type | Default | Description | +| :--------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--parallel-install-threads` | Integer | `10` | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. Pass this option with the value `0` to entirely disable parallel installation. | > **Note:** The `--require-poetry` runtime option is deprecated and will be removed in > version 1.0.0. Please set `require_poetry = true` in `tox.ini` for environments that > should fail if Poetry is not available. +> **Note:** The `--parallelize-locked-install` option is deprecated and will be removed in +> version 1.0.0. Please use the `--parallel-install-threads` option. + ### Errors There are several errors that the plugin can encounter for a test environment when Tox is From c4bf9bec24514cd259abe9d6f5cbb91f645d1fe7 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 16:27:56 -0400 Subject: [PATCH 3/8] Update tests to use new installer value --- tests/test_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_installer.py b/tests/test_installer.py index 45c9d68..75cc6ce 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -43,7 +43,7 @@ def test_parallelization(mock_venv, mock_poetry_factory): venv_sequential = tox.venv.VirtualEnv() start_sequential = time.time() - installer.install(poetry, venv_sequential, to_install, None) + installer.install(poetry, venv_sequential, to_install, 0) sequential = time.time() - start_sequential venv_parallel = tox.venv.VirtualEnv() From d5def209f28984ef46e28dc37e42ba9f7bc0418e Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 16:28:30 -0400 Subject: [PATCH 4/8] Bump feature version --- pyproject.toml | 2 +- tox_poetry_installer/__about__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 378c7a3..2097e4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tox-poetry-installer" -version = "0.7.0" +version = "0.8.0" license = "MIT" authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"] description = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile" diff --git a/tox_poetry_installer/__about__.py b/tox_poetry_installer/__about__.py index bdc1b5d..e839b42 100644 --- a/tox_poetry_installer/__about__.py +++ b/tox_poetry_installer/__about__.py @@ -1,7 +1,7 @@ # pylint: disable=missing-docstring __title__ = "tox-poetry-installer" __summary__ = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile" -__version__ = "0.7.0" +__version__ = "0.8.0" __url__ = "https://github.com/enpaul/tox-poetry-installer/" __license__ = "MIT" __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"] From 6b84764d5db801a813b6a620dd11e2ef9fb6a7e7 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 16:35:46 -0400 Subject: [PATCH 5/8] Add option to disable installation of project dependencies Fixes #53 --- tox_poetry_installer/hooks.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index 871c9f6..dcaa94e 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -58,6 +58,13 @@ def tox_addoption(parser: ToxParser): help="Automatically install all Poetry development dependencies to the environment", ) + parser.add_testenv_attribute( + name="install_project_deps", + type="bool", + default=True, + help="Automatically install all Poetry primary dependencies to the environment", + ) + parser.add_testenv_attribute( name="require_locked_deps", type="bool", @@ -140,7 +147,11 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional f"Identified {len(env_deps)} environment dependencies to install to env" ) - if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist: + if ( + not venv.envconfig.skip_install + and not venv.envconfig.config.skipsdist + and venv.envconfig.install_project_deps + ): project_deps = utilities.find_project_deps( packages, virtualenv, poetry, venv.envconfig.extras ) @@ -149,7 +160,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional ) else: project_deps = [] - logger.info("Env does not install project package, skipping") + logger.info("Env does not install project package dependencies, skipping") except exceptions.ToxPoetryInstallerException as err: venv.status = err.__class__.__name__ logger.error(str(err)) From 182fa2421436269b5222d13fc0d7ab2427a7bb79 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 16:37:10 -0400 Subject: [PATCH 6/8] Add docs for project deps config option --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9f5ba65..0ce8c42 100644 --- a/README.md +++ b/README.md @@ -194,12 +194,13 @@ configuration section. > test environments (for example, `testenv:foo`). To override this, specify the setting in > the child environment with a different value. -| Option | Type | Default | Description | -| :-------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `locked_deps` | List | `[]` | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included. | -| `require_locked_deps` | Boolean | False | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. | -| `install_dev_deps` | Boolean | False | Whether all of the Poetry dev-dependencies should be installed to the test environment. | -| `require_poetry` | Boolean | False | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail. | +| Option | Type | Default | Description | +| :--------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `locked_deps` | List | `[]` | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included. | +| `require_locked_deps` | Boolean | False | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. | +| `install_dev_deps` | Boolean | False | Whether all of the Poetry dev-dependencies should be installed to the test environment. | +| `install_project_deps` | Boolean | True | Whether all of the Poetry primary dependencies for the project package should be installed to the test environment. | +| `require_poetry` | Boolean | False | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail. | ### Runtime Options From 26bbe1372219596eca32b4dfaa0d44a14a0c8a35 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 22:16:49 -0400 Subject: [PATCH 7/8] Update changelog with version 0.8 --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f326e72..03057c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ See also: [Github Release Page](https://github.com/enpaul/tox-poetry-installer/releases). +## Version 0.8.0 + +View this release on: +[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.0), +[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.0/) + +- Add default installation of locked dependencies using thread workers, decreasing + environment provisioning times by ~90% +- Add runtime option `--parallel-install-threads` to support configuring the number of + worker threads for parallel dependency installation +- Add configuration option `install_project_deps` to support disabling the install of + project dependencies to an environment +- Deprecate runtime option `--parallelize-locked-install` + ## Version 0.7.0 View this release on: From 5188a30e77f5f9adc1abcba7ca55dcc724a6ec33 Mon Sep 17 00:00:00 2001 From: Ethan Paul <24588726+enpaul@users.noreply.github.com> Date: Wed, 5 May 2021 22:21:48 -0400 Subject: [PATCH 8/8] Update CI to drop deprecated runtime option --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 904b59e..30c31b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,7 @@ jobs: - name: Setup:env run: .github/scripts/setup-env.sh - name: Run:${{ matrix.python.toxenv }} - run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} Check: runs-on: ubuntu-latest steps: @@ -63,8 +63,8 @@ jobs: - name: Setup:env run: .github/scripts/setup-env.sh - name: Run:static - run: $HOME/ci/bin/tox -e static --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e static - name: Run:static-tests - run: $HOME/ci/bin/tox -e static-tests --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e static-tests - name: Run:security - run: $HOME/ci/bin/tox -e security --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e security