Update changelog with bugfix for #65

Update formatting

.github/workflows/ci.yaml

@@ -40,7 +40,7 @@ jobs:
       - name: Setup:env
         run: .github/scripts/setup-env.sh
       - name: Run:${{ matrix.python.toxenv }}
-        run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }} --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e ${{ matrix.python.toxenv }}
   Check:
     runs-on: ubuntu-latest
     steps:
@@ -63,8 +63,8 @@ jobs:
       - name: Setup:env
         run: .github/scripts/setup-env.sh
       - name: Run:static
-        run: $HOME/ci/bin/tox -e static --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e static
       - name: Run:static-tests
-        run: $HOME/ci/bin/tox -e static-tests --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e static-tests
       - name: Run:security
-        run: $HOME/ci/bin/tox -e security --parallelize-locked-install=10
+        run: $HOME/ci/bin/tox -e security

CHANGELOG.md

@@ -2,6 +2,40 @@
 
 See also: [Github Release Page](https://github.com/enpaul/tox-poetry-installer/releases).
 
+## Version 0.8.2
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.2),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.2/)
+
+- Improve debug-level logging for package installation, and time how long installing each
+  package takes. Contributed by [Rebecca
+  Turner](https://github.com/9999years).
+- Fix crash caused by the package-under-test depending on Poetry's unsafe dependencies ([#65](https://github.com/enpaul/tox-poetry-installer/issues/65))
+
+## Version 0.8.1
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.1),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.1/)
+
+- Fix unintuitive behavior of the `install_project_deps` option by ensuring the specified
+  value always causes the implied action
+
+## Version 0.8.0
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.0),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.0/)
+
+- Add default installation of locked dependencies using thread workers, decreasing
+  environment provisioning times by ~90%
+- Add runtime option `--parallel-install-threads` to support configuring the number of
+  worker threads for parallel dependency installation
+- Add configuration option `install_project_deps` to support disabling the install of
+  project dependencies to an environment
+- Deprecate runtime option `--parallelize-locked-install`
+
 ## Version 0.7.0
 
 View this release on:

LICENSE.md

@@ -1,4 +1,4 @@
-## Copyright 2020 Ethan Paul
+## Copyright 2020, 2021 Ethan Paul
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this
 software and associated documentation files (the "Software"), to deal in the Software
@@ -9,4 +9,9 @@ to whom the Software is furnished to do so, subject to the following conditions:
 The above copyright notice and this permission notice shall be included in all copies or
 substantial portions of the Software.
 
-**THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.**
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

README.md

@@ -80,7 +80,7 @@ adding the below to `tox.ini`, though this is also not recommended:
 
 ```ini
 requires =
-    tox-poetry-installer[poetry] == 0.7.0
+    tox-poetry-installer[poetry] == 0.8.0
 ```
 
 After installing, check that Tox recognizes the plugin by running
@@ -89,7 +89,7 @@ After installing, check that Tox recognizes the plugin by running
 ```
 3.20.0 imported from .venv/lib64/python3.8/site-packages/tox/__init__.py
 registered plugins:
-    tox-poetry-installer-0.7.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer.py
+    tox-poetry-installer-0.8.0 at .venv/lib64/python3.8/site-packages/tox_poetry_installer/__init__.py
 ```
 
 ### Quick Start
@@ -194,26 +194,30 @@ configuration section.
 > test environments (for example, `testenv:foo`). To override this, specify the setting in
 > the child environment with a different value.
 
-| Option                |  Type   | Default | Description                                                                                                                                                                                                                                                                                                                                                          |
-| :-------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `locked_deps`         |  List   |  `[]`   | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included.                                                                                                                                                                                          |
-| `require_locked_deps` | Boolean |  False  | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. |
-| `install_dev_deps`    | Boolean |  False  | Whether all of the Poetry dev-dependencies should be installed to the test environment.                                                                                                                                                                                                                                                                              |
-| `require_poetry`      | Boolean |  False  | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail.                                                                                            |
+| Option                 |  Type   | Default | Description                                                                                                                                                                                                                                                                                                                                                          |
+| :--------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `locked_deps`          |  List   |  `[]`   | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included.                                                                                                                                                                                          |
+| `require_locked_deps`  | Boolean |  False  | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. |
+| `install_dev_deps`     | Boolean |  False  | Whether all of the Poetry dev-dependencies should be installed to the test environment.                                                                                                                                                                                                                                                                              |
+| `install_project_deps` | Boolean |  True   | Whether all of the Poetry primary dependencies for the project package should be installed to the test environment.                                                                                                                                                                                                                                                  |
+| `require_poetry`       | Boolean |  False  | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail.                                                                                            |
 
 ### Runtime Options
 
 All arguments listed below can be passed to the `tox` command to modify runtime behavior
 of the plugin.
 
-| Argument                       |  Type   | Default | Description                                                                                                                                                                                                                                                                                            |
-| :----------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--parallelize-locked-install` | Integer |   `0`   | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. The default, `0`, disables the parallel install so that dependencies are installed sequentially. |
+| Argument                     |  Type   | Default | Description                                                                                                                                                                                                                                                                          |
+| :--------------------------- | :-----: | :-----: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--parallel-install-threads` | Integer |  `10`   | Number of worker threads to use to install dependencies in parallel. Installing in parallel with more threads can greatly speed up the install process, but can cause race conditions during install. Pass this option with the value `0` to entirely disable parallel installation. |
 
 > **Note:** The `--require-poetry` runtime option is deprecated and will be removed in
 > version 1.0.0. Please set `require_poetry = true` in `tox.ini` for environments that
 > should fail if Poetry is not available.
 
+> **Note:** The `--parallelize-locked-install` option is deprecated and will be removed in
+> version 1.0.0. Please use the `--parallel-install-threads` option.
+
 ### Errors
 
 There are several errors that the plugin can encounter for a test environment when Tox is
@@ -227,6 +231,7 @@ error will be set to one of the "Status" values below to indicate what the error
 | `LockedDepNotFoundError`        | Indicates that an item specified in the `locked_deps` config option does not match the name of a package in the Poetry lockfile.                                                                                                |
 | `LockedDepsRequiredError`       | Indicates that a test environment with the `require_locked_deps` config option set to `true` also specified unlocked dependencies using the [`deps`](https://tox.readthedocs.io/en/latest/config.html#conf-deps) config option. |
 | `PoetryNotInstalledError`       | Indicates that the `poetry` module could not be imported under the current runtime environment, and the `--require-poetry` flag was provided.                                                                                   |
+| `RequiresUnsafeDepError`        | Indicates that the package-under-test depends on a package that Poetry has classified as unsafe and cannot be installed.                                                                                                        |
 
 > **Note:** One or more of these errors can be caused by the `pyproject.toml` being out of
 > sync with the Poetry lockfile. If this is the case, than a warning will be logged when Tox

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tox-poetry-installer"
-version = "0.7.0"
+version = "0.8.2"
 license = "MIT"
 authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 description = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile"

tests/test_installer.py

@@ -43,7 +43,7 @@ def test_parallelization(mock_venv, mock_poetry_factory):
 
     venv_sequential = tox.venv.VirtualEnv()
     start_sequential = time.time()
-    installer.install(poetry, venv_sequential, to_install, None)
+    installer.install(poetry, venv_sequential, to_install, 0)
     sequential = time.time() - start_sequential
 
     venv_parallel = tox.venv.VirtualEnv()

tox_poetry_installer/__about__.py

@@ -1,7 +1,7 @@
 # pylint: disable=missing-docstring
 __title__ = "tox-poetry-installer"
 __summary__ = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile"
-__version__ = "0.7.0"
+__version__ = "0.8.2"
 __url__ = "https://github.com/enpaul/tox-poetry-installer/"
 __license__ = "MIT"
 __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]

tox_poetry_installer/constants.py

@@ -21,3 +21,6 @@ REPORTER_PREFIX: str = f"{__about__.__title__}:"
 
 # Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile
 UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"}
+
+# Number of threads to use for installing dependencies by default
+DEFAULT_INSTALL_THREADS: int = 10

tox_poetry_installer/exceptions.py

@@ -11,6 +11,7 @@ All exceptions should inherit from the common base exception :exc:`ToxPoetryInst
    +-- LockedDepNotFoundError
    +-- ExtraNotFoundError
    +-- LockedDepsRequiredError
+   +-- RequiresUnsafeDepError
 
 """
 
@@ -41,3 +42,7 @@ class ExtraNotFoundError(ToxPoetryInstallerException):
 
 class LockedDepsRequiredError(ToxPoetryInstallerException):
     """Environment cannot specify unlocked dependencies when locked dependencies are required"""
+
+
+class RequiresUnsafeDepError(ToxPoetryInstallerException):
+    """Package under test depends on an unsafe dependency and cannot be installed"""

tox_poetry_installer/hooks.py

@@ -12,6 +12,7 @@ from tox.config import Parser as ToxParser
 from tox.venv import VirtualEnv as ToxVirtualEnv
 
 from tox_poetry_installer import __about__
+from tox_poetry_installer import constants
 from tox_poetry_installer import exceptions
 from tox_poetry_installer import installer
 from tox_poetry_installer import logger
@@ -19,6 +20,66 @@ from tox_poetry_installer import utilities
 from tox_poetry_installer.datatypes import PackageMap
 
 
+def _postprocess_install_project_deps(
+    testenv_config, value: Optional[str]  # pylint: disable=unused-argument
+) -> Optional[bool]:
+    """An awful hack to patch on three-state boolean logic to a config parameter
+
+    .. warning: This logic should 100% be removed in the next feature release. It's here to work
+                around a bad design for now but should not persist.
+
+    The bug filed in `#61`_ is caused by a combination of poor design and attempted cleverness. The
+    name of the ``install_project_deps`` config option implies that it has ultimate control over
+    whether the project dependencies are installed to the testenv, but this is not actually correct.
+    What it actually allows the user to do is force the project dependencies to not be installed to
+    an environment that would otherwise install them. This was intended behavior, however the
+    intention was wrong.
+
+    .. _`#61`: https://github.com/enpaul/tox-poetry-installer/issues/61
+
+    In an effort to be clever the plugin automatically skips installing project dependencies when
+    the project package is not installed to the testenv (``skip_install = true``) or if packaging
+    as a whole is disabled (``skipsdist = true``). The intention of this behavior is to install only
+    the expected dependencies to a testenv and no more. However, this conflicts with the
+    ``install_project_deps`` config option, which cannot override this behavior because it defaults
+    to ``True``. In effect, ``install_project_deps = true`` in fact means "automatically
+    determine whether to install project dependencies" and ``install_project_deps = false`` means
+    "never install the project dependencies". This is not ideal and unintuitive.
+
+    To avoid having to make a breaking change this workaround has been added to support three-state
+    logic between ``True``, ``False``, and ``None``. The ``install_project_deps`` option is now
+    parsed by Tox as a string with a default value of ``None``. If the value is not ``None`` then
+    this post processing function will try to convert it to a boolean the same way that Tox's
+    `SectionReader.getbool()`_ method does, raising an error to mimic the default behavior if it
+    can't.
+
+    .. _`SectionReader.getbool()`: https://github.com/tox-dev/tox/blob/f8459218ee5ab5753321b3eb989b7beee5b391ad/src/tox/config/__init__.py#L1724
+
+    The three states for the ``install_project_deps`` setting are:
+    * ``None`` - User did not configure the setting, package dependency installation is
+      determined automatically
+    * ``True`` - User configured the setting to ``True``, package dependencies will be installed
+    * ``False`` - User configured the setting to ``False``, package dependencies will not be
+      installed
+
+    This config option should be deprecated with the 1.0.0 release and instead an option like
+    ``always_install_project_deps`` should be added which overrides the default determination and
+    just installs the project dependencies. The counterpart (``never_install_project_deps``)
+    shouldn't be needed, since I don't think there's a real use case for that.
+    """
+    if value is None:
+        return value
+
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+
+    raise tox.exception.ConfigError(
+        f"install_project_deps: boolean value '{value}' needs to be 'True' or 'False'"
+    )
+
+
 @tox.hookimpl
 def tox_addoption(parser: ToxParser):
     """Add required configuration options to the tox INI file
@@ -31,7 +92,7 @@ def tox_addoption(parser: ToxParser):
         "--require-poetry",
         action="store_true",
         dest="require_poetry",
-        help="Trigger a failure if Poetry is not available to Tox",
+        help="(deprecated) Trigger a failure if Poetry is not available to Tox",
     )
 
     parser.add_argument(
@@ -39,7 +100,15 @@ def tox_addoption(parser: ToxParser):
         type=int,
         dest="parallelize_locked_install",
         default=None,
-        help="Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel",
+        help="(deprecated) Number of worker threads to use for installing dependencies from the Poetry lockfile in parallel",
+    )
+
+    parser.add_argument(
+        "--parallel-install-threads",
+        type=int,
+        dest="parallel_install_threads",
+        default=constants.DEFAULT_INSTALL_THREADS,
+        help="Number of locked dependencies to install simultaneously; set to 0 to disable parallel installation",
     )
 
     parser.add_testenv_attribute(
@@ -49,6 +118,14 @@ def tox_addoption(parser: ToxParser):
         help="Automatically install all Poetry development dependencies to the environment",
     )
 
+    parser.add_testenv_attribute(
+        name="install_project_deps",
+        type="string",
+        default=None,
+        help="Automatically install all Poetry primary dependencies to the environment",
+        postprocess=_postprocess_install_project_deps,
+    )
+
     parser.add_testenv_attribute(
         name="require_locked_deps",
         type="bool",
@@ -82,13 +159,6 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
     :param action: Tox action object
     """
 
-    if venv.envconfig.config.option.require_poetry:
-        logger.warning(
-            "DEPRECATION WARNING: The '--require-poetry' runtime option is deprecated and will be "
-            "removed in version 1.0.0. Please update test environments that require Poetry to "
-            "set the 'require_poetry = true' option in tox.ini"
-        )
-
     try:
         poetry = utilities.check_preconditions(venv, action)
     except exceptions.SkipEnvironment as err:
@@ -138,7 +208,15 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
             f"Identified {len(env_deps)} environment dependencies to install to env"
         )
 
-        if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist:
+        install_project_deps = (
+            venv.envconfig.install_project_deps
+            if venv.envconfig.install_project_deps is not None
+            else (
+                not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist
+            )
+        )
+
+        if install_project_deps:
             project_deps = utilities.find_project_deps(
                 packages, virtualenv, poetry, venv.envconfig.extras
             )
@@ -147,7 +225,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
             )
         else:
             project_deps = []
-            logger.info("Env does not install project package, skipping")
+            logger.info("Env does not install project package dependencies, skipping")
     except exceptions.ToxPoetryInstallerException as err:
         venv.status = err.__class__.__name__
         logger.error(str(err))
@@ -158,11 +236,18 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         raise err
 
     dependencies = dev_deps + env_deps + project_deps
-    log_parallel = (
-        f" (using {venv.envconfig.config.option.parallelize_locked_install} threads)"
-        if venv.envconfig.config.option.parallelize_locked_install
-        else ""
-    )
+    if (
+        venv.envconfig.config.option.parallel_install_threads
+        != constants.DEFAULT_INSTALL_THREADS
+    ):
+        parallel_threads = venv.envconfig.config.option.parallel_install_threads
+    else:
+        parallel_threads = (
+            venv.envconfig.config.option.parallelize_locked_install
+            if venv.envconfig.config.option.parallelize_locked_install is not None
+            else constants.DEFAULT_INSTALL_THREADS
+        )
+    log_parallel = f" (using {parallel_threads} threads)" if parallel_threads else ""
 
     action.setactivity(
         __about__.__title__,
@@ -172,7 +257,7 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         poetry,
         venv,
         dependencies,
-        venv.envconfig.config.option.parallelize_locked_install,
+        parallel_threads,
     )
 
     return venv.envconfig.require_locked_deps or None

tox_poetry_installer/installer.py

@@ -5,7 +5,7 @@
 import concurrent.futures
 import contextlib
 import typing
-from typing import Optional
+from datetime import datetime
 from typing import Sequence
 from typing import Set
 
@@ -23,7 +23,7 @@ def install(
     poetry: "_poetry.Poetry",
     venv: ToxVirtualEnv,
     packages: Sequence[PoetryPackage],
-    parallels: Optional[int] = None,
+    parallels: int = 0,
 ):
     """Install a bunch of packages to a virtualenv
 
@@ -47,6 +47,13 @@ def install(
 
     installed: Set[PoetryPackage] = set()
 
+    def logged_install(dependency: PoetryPackage) -> None:
+        start = datetime.now()
+        logger.debug(f"Installing {dependency}")
+        pip.install(dependency)
+        end = datetime.now()
+        logger.debug(f"Finished installing {dependency} in {end - start}")
+
     @contextlib.contextmanager
     def _optional_parallelize():
         """A bit of cheat, really
@@ -55,7 +62,7 @@ def install(
         enables/disables the usage of the parallel thread pooler depending on the value of
         the ``parallels`` parameter.
         """
-        if parallels:
+        if parallels > 0:
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=parallels
             ) as executor:
@@ -67,8 +74,8 @@ def install(
         for dependency in packages:
             if dependency not in installed:
                 installed.add(dependency)
-                logger.debug(f"Installing {dependency}")
-                executor(pip.install, dependency)
+                logger.debug(f"Queuing {dependency}")
+                executor(logged_install, dependency)
             else:
                 logger.debug(f"Skipping {dependency}, already installed")
         logger.debug("Waiting for installs to finish...")

tox_poetry_installer/utilities.py

@@ -43,6 +43,19 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poet
             f"Skipping isolated packaging build env '{action.name}'"
         )
 
+    if venv.envconfig.config.option.require_poetry:
+        logger.warning(
+            "DEPRECATION: The '--require-poetry' runtime option is deprecated and will be "
+            "removed in version 1.0.0. Please update test environments that require Poetry to "
+            "set the 'require_poetry = true' option in tox.ini"
+        )
+
+    if venv.envconfig.config.option.parallelize_locked_install is not None:
+        logger.warning(
+            "DEPRECATION: The '--parallelize-locked-install' option is deprecated and will "
+            "be removed in version 1.0.0. Please use the '--parallel-install-threads' option."
+        )
+
     from tox_poetry_installer import _poetry
 
     try:
@@ -153,6 +166,11 @@ def find_project_deps(
     :param extras: Sequence of extra names to include the dependencies of
     """
 
+    if any(dep.name in constants.UNSAFE_PACKAGES for dep in poetry.package.requires):
+        raise exceptions.RequiresUnsafeDepError(
+            f"Project package requires one or more unsafe dependencies ({', '.join(constants.UNSAFE_PACKAGES)}) which cannot be installed with Poetry"
+        )
+
     base_deps: List[PoetryPackage] = [
         packages[item.name]
         for item in poetry.package.requires