Merge pull request #62 from enpaul/enp/61

Add three state boolean logic for install_project_deps config

.github/workflows/ci.yaml

@@ -40,7 +40,7 @@ jobs:
       - name: Setup:env
         run: .github/scripts/setup-env.sh
       - name: Run:${{ matrix.python.toxenv }}
-        run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }}
   Check:
     runs-on: ubuntu-latest
     steps:
@@ -63,8 +63,8 @@ jobs:
       - name: Setup:env
         run: .github/scripts/setup-env.sh
       - name: Run:static
-        run: $HOME/ci/bin/tox -e static --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e static
       - name: Run:static-tests
-        run: $HOME/ci/bin/tox -e static-tests --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e static-tests
       - name: Run:security
-        run: $HOME/ci/bin/tox -e security --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e security

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 See also: [Github Release Page](https://github.com/enpaul/tox-poetry-installer/releases).
 
+## Version 0.8.0
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.0),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.0/)
+
+- Add default installation of locked dependencies using thread workers, decreasing
+  environment provisioning times by ~90%
+- Add runtime option `--parallel-install-threads` to support configuring the number of
+  worker threads for parallel dependency installation
+- Add configuration option `install_project_deps` to support disabling the install of
+  project dependencies to an environment
+- Deprecate runtime option `--parallelize-locked-install`
+
 ## Version 0.7.0
 
 View this release on:

README.md

@@ -80,7 +80,7 @@ adding the below to `tox.ini`, though this is also not recommended:
 
 ```ini
 requires =
-    tox-poetry-installer[poetry] == 0.7.0
+    tox-poetry-installer[poetry] == 0.8.0
 ```
 
 After installing, check that Tox recognizes the plugin by running
@@ -89,7 +89,7 @@ After installing, check that Tox recognizes the plugin by running
 ```
 3.20.0 imported from .venv/lib64/python3.8/site-packages/tox/__init__.py
 registered plugins:
-    tox-poetry-installer-0.7.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer.py
+    tox-poetry-installer-0.8.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer/__init__.py
 ```
 
 ### Quick Start
@@ -194,26 +194,30 @@ configuration section.
 > test environments (for example, `testenv:foo`). To override this, specify the setting in
 > the child environment with a different value.
 
-| Option                |  Type   | Default | Description                                                                                                                                                                                                                                                                                                                                                          |
-| :-------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `locked_deps`         |  List   |  `[]`   | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included.                                                                                                                                                                                          |
-| `require_locked_deps` | Boolean |  False  | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. |
-| `install_dev_deps`    | Boolean |  False  | Whether all of the Poetry dev-dependencies should be installed to the test environment.                                                                                                                                                                                                                                                                              |
-| `require_poetry`      | Boolean |  False  | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail.                                                                                            |
+| Option                 |  Type   | Default | Description                                                                                                                                                                                                                                                                                                                                                          |
+| :--------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `locked_deps`          |  List   |  `[]`   | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included.                                                                                                                                                                                          |
+| `require_locked_deps`  | Boolean |  False  | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. |
+| `install_dev_deps`     | Boolean |  False  | Whether all of the Poetry dev-dependencies should be installed to the test environment.                                                                                                                                                                                                                                                                              |
+| `install_project_deps` | Boolean |  True   | Whether all of the Poetry primary dependencies for the project package should be installed to the test environment.                                                                                                                                                                                                                                                  |
+| `require_poetry`       | Boolean |  False  | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail.                                                                                            |
 
 ### Runtime Options
 
 All arguments listed below can be passed to the `tox` command to modify runtime behavior
 of the plugin.
 
-| Argument                       |  Type   | Default | Description                                                                                                                                                                                                                                                                                            |
-| :----------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--parallelize-locked-install` | Integer |   `0`   | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. The default, `0`, disables the parallel install so that dependencies are installed sequentially. |
+| Argument                     |  Type   | Default | Description                                                                                                                                                                                                                                                                          |
+| :--------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--parallel-install-threads` | Integer |  `10`   | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. Pass this option with the value `0` to entirely disable parallel installation. |
 
 > **Note:** The `--require-poetry` runtime option is deprecated and will be removed in
 > version 1.0.0. Please set `require_poetry = true` in `tox.ini` for environments that
 > should fail if Poetry is not available.
 
+> **Note:** The `--parallelize-locked-install` option is deprecated and will be removed in
+> version 1.0.0. Please use the `--parallel-install-threads` option.
+
 ### Errors
 
 There are several errors that the plugin can encounter for a test environment when Tox is

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tox-poetry-installer"
-version = "0.7.0"
+version = "0.8.1"
 license = "MIT"
 authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 description = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile"

tests/test_installer.py

@@ -43,7 +43,7 @@ def test_parallelization(mock_venv, mock_poetry_factory):
 
     venv_sequential = tox.venv.VirtualEnv()
     start_sequential = time.time()
-    installer.install(poetry, venv_sequential, to_install, None)
+    installer.install(poetry, venv_sequential, to_install, 0)
     sequential = time.time() - start_sequential
 
     venv_parallel = tox.venv.VirtualEnv()

tox_poetry_installer/__about__.py

@@ -1,7 +1,7 @@
 # pylint: disable=missing-docstring
 __title__ = "tox-poetry-installer"
 __summary__ = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile"
-__version__ = "0.7.0"
+__version__ = "0.8.1"
 __url__ = "https://github.com/enpaul/tox-poetry-installer/"
 __license__ = "MIT"
 __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]

tox_poetry_installer/constants.py

@@ -21,3 +21,6 @@ REPORTER_PREFIX: str = f"{__about__.__title__}:"
 
 # Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile
 UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"}
+
+# Number of threads to use for installing dependencies by default
+DEFAULT_INSTALL_THREADS: int = 10

tox_poetry_installer/hooks.py

@@ -12,6 +12,7 @@ from tox.config import Parser as ToxParser
 from tox.venv import VirtualEnv as ToxVirtualEnv
 
 from tox_poetry_installer import __about__
+from tox_poetry_installer import constants
 from tox_poetry_installer import exceptions
 from tox_poetry_installer import installer
 from tox_poetry_installer import logger
@@ -19,6 +20,66 @@ from tox_poetry_installer import utilities
 from tox_poetry_installer.datatypes import PackageMap
 
 
+def _postprocess_install_project_deps(
+    testenv_config, value: Optional[str]  # pylint: disable=unused-argument
+) -> Optional[bool]:
+    """An awful hack to patch on three-state boolean logic to a config parameter
+
+    .. warning: This logic should 100% be removed in the next feature release. It's here to work
+                around a bad design for now but should not persist.
+
+    The bug filed in `#61`_ is caused by a combination of poor design and attempted cleverness. The
+    name of the ``install_project_deps`` config option implies that it has ultimate control over
+    whether the project dependencies are installed to the testenv, but this is not actually correct.
+    What it actually allows the user to do is force the project dependencies to not be installed to
+    an environment that would otherwise install them. This was intended behavior, however the
+    intention was wrong.
+
+    .. _`#61`: https://github.com/enpaul/tox-poetry-installer/issues/61
+
+    In an effort to be clever the plugin automatically skips installing project dependencies when
+    the project package is not installed to the testenv (``skip_install = true``) or if packaging
+    as a whole is disabled (``skipsdist = true``). The intention of this behavior is to install only
+    the expected dependencies to a testenv and no more. However, this conflicts with the
+    ``install_project_deps`` config option, which cannot override this behavior because it defaults
+    to ``True``. In effect, ``install_project_deps = true`` in fact means "automatically
+    determine whether to install project dependencies" and ``install_project_deps = false`` means
+    "never install the project dependencies". This is not ideal and unintuitive.
+
+    To avoid having to make a breaking change this workaround has been added to support three-state
+    logic between ``True``, ``False``, and ``None``. The ``install_project_deps`` option is now
+    parsed by Tox as a string with a default value of ``None``. If the value is not ``None`` then
+    this post processing function will try to convert it to a boolean the same way that Tox's
+    `SectionReader.getbool()`_ method does, raising an error to mimic the default behavior if it
+    can't.
+
+    .. _`SectionReader.getbool()`: https://github.com/tox-dev/tox/blob/f8459218ee5ab5753321b3eb989b7beee5b391ad/src/tox/config/__init__.py#L1724
+
+    The three states for the ``install_project_deps`` setting are:
+    * ``None`` - User did not configure the setting, package dependency installation is
+      determined automatically
+    * ``True`` - User configured the setting to ``True``, package dependencies will be installed
+    * ``False`` - User configured the setting to ``False``, package dependencies will not be
+      installed
+
+    This config option should be deprecated with the 1.0.0 release and instead an option like
+    ``always_install_project_deps`` should be added which overrides the default determination and
+    just installs the project dependencies. The counterpart (``never_install_project_deps``)
+    shouldn't be needed, since I don't think there's a real use case for that.
+    """
+    if value is None:
+        return value
+
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+
+    raise tox.exception.ConfigError(
+        f"install_project_deps: boolean value '{value}' needs to be 'True' or 'False'"
+    )
+
+
 @tox.hookimpl
 def tox_addoption(parser: ToxParser):
     """Add required configuration options to the tox INI file
@@ -31,7 +92,7 @@ def tox_addoption(parser: ToxParser):
         "--require-poetry",
         action="store_true",
         dest="require_poetry",
-        help="Trigger a failure if Poetry is not available to Tox",
+        help="(deprecated) Trigger a failure if Poetry is not available to Tox",
     )
 
     parser.add_argument(
@@ -39,7 +100,15 @@ def tox_addoption(parser: ToxParser):
         type=int,
         dest="parallelize_locked_install",
         default=None,
-        help="Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel",
+        help="(deprecated) Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel",
+    )
+
+    parser.add_argument(
+        "--parallel-install-threads",
+        type=int,
+        dest="parallel_install_threads",
+        default=constants.DEFAULT_INSTALL_THREADS,
+        help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
     )
 
     parser.add_testenv_attribute(
@@ -49,6 +118,14 @@ def tox_addoption(parser: ToxParser):
         help="Automatically install all Poetry development dependencies to the environment",
     )
 
+    parser.add_testenv_attribute(
+        name="install_project_deps",
+        type="string",
+        default=None,
+        help="Automatically install all Poetry primary dependencies to the environment",
+        postprocess=_postprocess_install_project_deps,
+    )
+
     parser.add_testenv_attribute(
         name="require_locked_deps",
         type="bool",
@@ -82,13 +159,6 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
     :param action: Tox action object
     """
 
-    if venv.envconfig.config.option.require_poetry:
-        logger.warning(
-            "DEPRECATION WARNING: The '--require-poetry' runtime option is deprecated and will be "
-            "removed in version 1.0.0. Please update test environments that require Poetry to "
-            "set the 'require_poetry = true' option in tox.ini"
-        )
-
     try:
         poetry = utilities.check_preconditions(venv, action)
     except exceptions.SkipEnvironment as err:
@@ -138,7 +208,15 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
             f"Identified {len(env_deps)} environment dependencies to install to env"
         )
 
-        if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist:
+        install_project_deps = (
+            venv.envconfig.install_project_deps
+            if venv.envconfig.install_project_deps is not None
+            else (
+                not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist
+            )
+        )
+
+        if install_project_deps:
             project_deps = utilities.find_project_deps(
                 packages, virtualenv, poetry, venv.envconfig.extras
             )
@@ -147,7 +225,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
             )
         else:
             project_deps = []
-            logger.info("Env does not install project package, skipping")
+            logger.info("Env does not install project package dependencies, skipping")
     except exceptions.ToxPoetryInstallerException as err:
         venv.status = err.__class__.__name__
         logger.error(str(err))
@@ -158,11 +236,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         raise err
 
     dependencies = dev_deps + env_deps + project_deps
-    log_parallel = (
-        f" (using {venv.envconfig.config.option.parallelize_locked_install} threads)"
-        if venv.envconfig.config.option.parallelize_locked_install
-        else ""
-    )
+    if (
+        venv.envconfig.config.option.parallel_install_threads
+        != constants.DEFAULT_INSTALL_THREADS
+    ):
+        parallel_threads = venv.envconfig.config.option.parallel_install_threads
+    else:
+        parallel_threads = (
+            venv.envconfig.config.option.parallelize_locked_install
+            if venv.envconfig.config.option.parallelize_locked_install is not None
+            else constants.DEFAULT_INSTALL_THREADS
+        )
+    log_parallel = f" (using {parallel_threads} threads)" if parallel_threads else ""
 
     action.setactivity(
         __about__.__title__,
@@ -172,7 +257,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         poetry,
         venv,
         dependencies,
-        venv.envconfig.config.option.parallelize_locked_install,
+        parallel_threads,
     )
 
     return venv.envconfig.require_locked_deps or None

tox_poetry_installer/installer.py

@@ -5,7 +5,6 @@
 import concurrent.futures
 import contextlib
 import typing
-from typing import Optional
 from typing import Sequence
 from typing import Set
 
@@ -23,7 +22,7 @@ def install(
     poetry: "_poetry.Poetry",
     venv: ToxVirtualEnv,
     packages: Sequence[PoetryPackage],
-    parallels: Optional[int] = None,
+    parallels: int = 0,
 ):
     """Install a bunch of packages to a virtualenv
 
@@ -55,7 +54,7 @@ def install(
         enables/disables the usage of the parallel thread pooler depending on the value of
         the ``parallels`` parameter.
         """
-        if parallels:
+        if parallels > 0:
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=parallels
             ) as executor:

tox_poetry_installer/utilities.py

@@ -43,6 +43,19 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poet
             f"Skipping isolated packaging build env '{action.name}'"
         )
 
+    if venv.envconfig.config.option.require_poetry:
+        logger.warning(
+            "DEPRECATION: The '--require-poetry' runtime option is deprecated and will be "
+            "removed in version 1.0.0. Please update test environments that require Poetry to "
+            "set the 'require_poetry = true' option in tox.ini"
+        )
+
+    if venv.envconfig.config.option.parallelize_locked_install is not None:
+        logger.warning(
+            "DEPRECATION: The '--parallelize-locked-install' option is deprecated and will "
+            "be removed in version 1.0.0. Please use the '--parallel-install-threads' option."
+        )
+
     from tox_poetry_installer import _poetry
 
     try: