diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 904b59e..30c31b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,7 +40,7 @@ jobs: - name: Setup:env run: .github/scripts/setup-env.sh - name: Run:${{ matrix.python.toxenv }} - run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} Check: runs-on: ubuntu-latest steps: @@ -63,8 +63,8 @@ jobs: - name: Setup:env run: .github/scripts/setup-env.sh - name: Run:static - run: $HOME/ci/bin/tox -e static --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e static - name: Run:static-tests - run: $HOME/ci/bin/tox -e static-tests --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e static-tests - name: Run:security - run: $HOME/ci/bin/tox -e security --parallelize-locked-install=10 + run: $HOME/ci/bin/tox -e security diff --git a/CHANGELOG.md b/CHANGELOG.md index f326e72..03057c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ See also: [Github Release Page](https://github.com/enpaul/tox-poetry-installer/releases). +## Version 0.8.0 + +View this release on: +[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.0), +[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.0/) + +- Add default installation of locked dependencies using thread workers, decreasing + environment provisioning times by ~90% +- Add runtime option `--parallel-install-threads` to support configuring the number of + worker threads for parallel dependency installation +- Add configuration option `install_project_deps` to support disabling the install of + project dependencies to an environment +- Deprecate runtime option `--parallelize-locked-install` + ## Version 0.7.0 View this release on: diff --git a/README.md b/README.md index 49891a4..0ce8c42 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ adding the below to `tox.ini`, though this is also not recommended: ```ini requires = - tox-poetry-installer[poetry] == 0.7.0 + tox-poetry-installer[poetry] == 0.8.0 ``` After installing, check that Tox recognizes the plugin by running @@ -89,7 +89,7 @@ After installing, check that Tox recognizes the plugin by running ``` 3.20.0 imported from .venv/lib64/python3.8/site-packages/tox/__init__.py registered plugins: - tox-poetry-installer-0.7.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer.py + tox-poetry-installer-0.8.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer/__init__.py ``` ### Quick Start @@ -194,26 +194,30 @@ configuration section. > test environments (for example, `testenv:foo`). To override this, specify the setting in > the child environment with a different value. -| Option | Type | Default | Description | -| :-------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `locked_deps` | List | `[]` | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included. | -| `require_locked_deps` | Boolean | False | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. | -| `install_dev_deps` | Boolean | False | Whether all of the Poetry dev-dependencies should be installed to the test environment. | -| `require_poetry` | Boolean | False | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail. | +| Option | Type | Default | Description | +| :--------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `locked_deps` | List | `[]` | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included. | +| `require_locked_deps` | Boolean | False | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. | +| `install_dev_deps` | Boolean | False | Whether all of the Poetry dev-dependencies should be installed to the test environment. | +| `install_project_deps` | Boolean | True | Whether all of the Poetry primary dependencies for the project package should be installed to the test environment. | +| `require_poetry` | Boolean | False | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail. | ### Runtime Options All arguments listed below can be passed to the `tox` command to modify runtime behavior of the plugin. -| Argument | Type | Default | Description | -| :----------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--parallelize-locked-install` | Integer | `0` | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. The default, `0`, disables the parallel install so that dependencies are installed sequentially. | +| Argument | Type | Default | Description | +| :--------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--parallel-install-threads` | Integer | `10` | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. Pass this option with the value `0` to entirely disable parallel installation. | > **Note:** The `--require-poetry` runtime option is deprecated and will be removed in > version 1.0.0. Please set `require_poetry = true` in `tox.ini` for environments that > should fail if Poetry is not available. +> **Note:** The `--parallelize-locked-install` option is deprecated and will be removed in +> version 1.0.0. Please use the `--parallel-install-threads` option. + ### Errors There are several errors that the plugin can encounter for a test environment when Tox is diff --git a/pyproject.toml b/pyproject.toml index 378c7a3..2097e4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tox-poetry-installer" -version = "0.7.0" +version = "0.8.0" license = "MIT" authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"] description = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile" diff --git a/tests/test_installer.py b/tests/test_installer.py index 45c9d68..75cc6ce 100644 --- a/tests/test_installer.py +++ b/tests/test_installer.py @@ -43,7 +43,7 @@ def test_parallelization(mock_venv, mock_poetry_factory): venv_sequential = tox.venv.VirtualEnv() start_sequential = time.time() - installer.install(poetry, venv_sequential, to_install, None) + installer.install(poetry, venv_sequential, to_install, 0) sequential = time.time() - start_sequential venv_parallel = tox.venv.VirtualEnv() diff --git a/tox_poetry_installer/__about__.py b/tox_poetry_installer/__about__.py index bdc1b5d..e839b42 100644 --- a/tox_poetry_installer/__about__.py +++ b/tox_poetry_installer/__about__.py @@ -1,7 +1,7 @@ # pylint: disable=missing-docstring __title__ = "tox-poetry-installer" __summary__ = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile" -__version__ = "0.7.0" +__version__ = "0.8.0" __url__ = "https://github.com/enpaul/tox-poetry-installer/" __license__ = "MIT" __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"] diff --git a/tox_poetry_installer/constants.py b/tox_poetry_installer/constants.py index 3be4fd8..ccfc6f3 100644 --- a/tox_poetry_installer/constants.py +++ b/tox_poetry_installer/constants.py @@ -21,3 +21,6 @@ REPORTER_PREFIX: str = f"{__about__.__title__}:" # Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"} + +# Number of threads to use for installing dependencies by default +DEFAULT_INSTALL_THREADS: int = 10 diff --git a/tox_poetry_installer/hooks.py b/tox_poetry_installer/hooks.py index 7c880ee..dcaa94e 100644 --- a/tox_poetry_installer/hooks.py +++ b/tox_poetry_installer/hooks.py @@ -12,6 +12,7 @@ from tox.config import Parser as ToxParser from tox.venv import VirtualEnv as ToxVirtualEnv from tox_poetry_installer import __about__ +from tox_poetry_installer import constants from tox_poetry_installer import exceptions from tox_poetry_installer import installer from tox_poetry_installer import logger @@ -31,7 +32,7 @@ def tox_addoption(parser: ToxParser): "--require-poetry", action="store_true", dest="require_poetry", - help="Trigger a failure if Poetry is not available to Tox", + help="(deprecated) Trigger a failure if Poetry is not available to Tox", ) parser.add_argument( @@ -39,7 +40,15 @@ def tox_addoption(parser: ToxParser): type=int, dest="parallelize_locked_install", default=None, - help="Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel", + help="(deprecated) Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel", + ) + + parser.add_argument( + "--parallel-install-threads", + type=int, + dest="parallel_install_threads", + default=constants.DEFAULT_INSTALL_THREADS, + help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation", ) parser.add_testenv_attribute( @@ -49,6 +58,13 @@ def tox_addoption(parser: ToxParser): help="Automatically install all Poetry development dependencies to the environment", ) + parser.add_testenv_attribute( + name="install_project_deps", + type="bool", + default=True, + help="Automatically install all Poetry primary dependencies to the environment", + ) + parser.add_testenv_attribute( name="require_locked_deps", type="bool", @@ -82,13 +98,6 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional :param action: Tox action object """ - if venv.envconfig.config.option.require_poetry: - logger.warning( - "DEPRECATION WARNING: The '--require-poetry' runtime option is deprecated and will be " - "removed in version 1.0.0. Please update test environments that require Poetry to " - "set the 'require_poetry = true' option in tox.ini" - ) - try: poetry = utilities.check_preconditions(venv, action) except exceptions.SkipEnvironment as err: @@ -138,7 +147,11 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional f"Identified {len(env_deps)} environment dependencies to install to env" ) - if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist: + if ( + not venv.envconfig.skip_install + and not venv.envconfig.config.skipsdist + and venv.envconfig.install_project_deps + ): project_deps = utilities.find_project_deps( packages, virtualenv, poetry, venv.envconfig.extras ) @@ -147,7 +160,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional ) else: project_deps = [] - logger.info("Env does not install project package, skipping") + logger.info("Env does not install project package dependencies, skipping") except exceptions.ToxPoetryInstallerException as err: venv.status = err.__class__.__name__ logger.error(str(err)) @@ -158,11 +171,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional raise err dependencies = dev_deps + env_deps + project_deps - log_parallel = ( - f" (using {venv.envconfig.config.option.parallelize_locked_install} threads)" - if venv.envconfig.config.option.parallelize_locked_install - else "" - ) + if ( + venv.envconfig.config.option.parallel_install_threads + != constants.DEFAULT_INSTALL_THREADS + ): + parallel_threads = venv.envconfig.config.option.parallel_install_threads + else: + parallel_threads = ( + venv.envconfig.config.option.parallelize_locked_install + if venv.envconfig.config.option.parallelize_locked_install is not None + else constants.DEFAULT_INSTALL_THREADS + ) + log_parallel = f" (using {parallel_threads} threads)" if parallel_threads else "" action.setactivity( __about__.__title__, @@ -172,7 +192,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional poetry, venv, dependencies, - venv.envconfig.config.option.parallelize_locked_install, + parallel_threads, ) return venv.envconfig.require_locked_deps or None diff --git a/tox_poetry_installer/installer.py b/tox_poetry_installer/installer.py index 1685ea0..bbde07f 100644 --- a/tox_poetry_installer/installer.py +++ b/tox_poetry_installer/installer.py @@ -5,7 +5,6 @@ import concurrent.futures import contextlib import typing -from typing import Optional from typing import Sequence from typing import Set @@ -23,7 +22,7 @@ def install( poetry: "_poetry.Poetry", venv: ToxVirtualEnv, packages: Sequence[PoetryPackage], - parallels: Optional[int] = None, + parallels: int = 0, ): """Install a bunch of packages to a virtualenv @@ -55,7 +54,7 @@ def install( enables/disables the usage of the parallel thread pooler depending on the value of the ``parallels`` parameter. """ - if parallels: + if parallels > 0: with concurrent.futures.ThreadPoolExecutor( max_workers=parallels ) as executor: diff --git a/tox_poetry_installer/utilities.py b/tox_poetry_installer/utilities.py index 6f7ac80..87e1d98 100644 --- a/tox_poetry_installer/utilities.py +++ b/tox_poetry_installer/utilities.py @@ -43,6 +43,19 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poet f"Skipping isolated packaging build env '{action.name}'" ) + if venv.envconfig.config.option.require_poetry: + logger.warning( + "DEPRECATION: The '--require-poetry' runtime option is deprecated and will be " + "removed in version 1.0.0. Please update test environments that require Poetry to " + "set the 'require_poetry = true' option in tox.ini" + ) + + if venv.envconfig.config.option.parallelize_locked_install is not None: + logger.warning( + "DEPRECATION: The '--parallelize-locked-install' option is deprecated and will " + "be removed in version 1.0.0. Please use the '--parallel-install-threads' option." + ) + from tox_poetry_installer import _poetry try: