Merge pull request #16 from enpaul/enp/fix

Fix support for extras installation

.github/workflows/ci.yaml

@@ -0,0 +1,41 @@
+---
+name: CI
+on:
+  pull_request:
+    types: ["opened", "synchronize"]
+  push:
+    branches: ["devel"]
+jobs:
+  Test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python:
+          - version: 3.6
+            toxenv: py36
+          - version: 3.7
+            toxenv: py37
+          - version: 3.8
+            toxenv: py38
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python.version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python.version }}
+      - name: Install tox
+        run: pip install "tox>=3.20.0,<3.21.0" --upgrade
+      - name: Run tests via ${{ matrix.python.toxenv }}
+        run: tox -e ${{ matrix.python.toxenv }}
+  Check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install tox requirements
+        run: pip install "tox>=3.20.0,<3.21.0" --upgrade
+      - name: Run meta checks
+        run: tox -e static -e static-tests -e security

.pre-commit-config.yaml

@@ -4,14 +4,14 @@ repos:
     rev: 20.8b1
     hooks:
       - id: black
-        language_version: python3.7
+        language_version: python3
 
   - repo: https://github.com/asottile/blacken-docs
     rev: v0.5.0
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==20.8b1]
-        language_version: python3.7
+        language_version: python3
 
   - repo: https://github.com/asottile/reorder_python_imports
     rev: v2.3.5

README.md

@@ -1,19 +1,23 @@
 # tox-poetry-installer
 
 A plugin for [Tox](https://tox.readthedocs.io/en/latest/) that allows test environment
-dependencies to be installed using [Poetry](https://python-poetry.org/) using its lockfile.
+dependencies to be installed using [Poetry](https://python-poetry.org/) from its lockfile.
 
-⚠️ **This project is alpha software and should not be used in a production capacity** ⚠️
+⚠️ **This project is alpha software and should not be used in production environments** ⚠️
 
-![image](https://img.shields.io/pypi/l/tox-poetry-installer)
+[![ci-status](https://github.com/enpaul/tox-poetry-installer/workflows/CI/badge.svg?event=push)](https://github.com/enpaul/tox-poetry-installer/actions)
-![image](https://img.shields.io/pypi/v/tox-poetry-installer)
+[![license](https://img.shields.io/pypi/l/tox-poetry-installer)](https://opensource.org/licenses/MIT)
-![image](https://img.shields.io/pypi/pyversions/tox-poetry-installer)
+[![pypi-version](https://img.shields.io/pypi/v/tox-poetry-installer)](https://pypi.org/project/tox-poetry-installer/)
+[![python-versions](https://img.shields.io/pypi/pyversions/tox-poetry-installer)](https://www.python.org)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 **Documentation**
 
-* [Installation and Usage](#installation-and-usage)
+* [Installation](#installation)
-* [Limitations](#limitations)
+* [Quick Start](#quick-start)
-* [Why would I use this?](#what-problems-does-this-solve) (What problems does this solve?)
+* [Usage Examples](#usage-examples)
+* [Known Drawbacks and Problems](#known-drawbacks-and-problems)
+* [Why would I use this?](#why-would-i-use-this) (What problems does this solve?)
 * [Developing](#developing)
 * [Contributing](#contributing)
 * [Roadmap](#roadmap)
@@ -23,85 +27,260 @@ dependencies to be installed using [Poetry](https://python-poetry.org/) using it
 Related resources:
 * [Poetry Python Project Manager](https://python-poetry.org/)
 * [Tox Automation Project](https://tox.readthedocs.io/en/latest/)
-* [Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html)
+* [Poetry Dev-Dependencies Tox Plugin](https://github.com/sinoroc/tox-poetry-dev-dependencies)
+* [Poetry Tox Plugin](https://github.com/tkukushkin/tox-poetry)
+* [Other Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html)
 
 
-## Installation and Usage
+## Installation
 
-1. Install the plugin from PyPI:
+Add the plugin as a development dependency of a Poetry project:
 
 ```
-poetry add tox-poetry-installer --dev
+~ $: poetry add tox-poetry-installer --dev
 ```
 
-2. Remove all version specifications from the environment dependencies in `tox.ini`:
+Confirm that the plugin is installed, and Tox recognizes it, by checking the Tox version:
+
+```
+~ $: poetry run tox --version
+3.20.0 imported from .venv/lib64/python3.8/site-packages/tox/__init__.py
+registered plugins:
+    tox-poetry-installer-0.2.2 at .venv/lib64/python3.8/site-packages/tox_poetry_installer.py
+```
+
+If using Pip, ensure that the plugin is installed to the same environment as Tox:
+
+```
+# Calling the virtualenv's 'pip' binary directly will cause pip to install to that virtualenv
+~ $: /path/to/my/automation/virtualenv/bin/pip install tox
+~ $: /path/to/my/automation/virtualenv/bin/pip install tox-poetry-installer
+```
+
+
+## Quick Start
+
+To require a Tox environment install all it's dependencies from the Poetry lockfile, add the
+`require_locked_deps = true` option to the environment configuration and remove all version
+specifiers from the dependency list. The versions to install will be taken from the lockfile
+directly:
 
 ```ini
-# This...
 [testenv]
-description = My cool test environment
+description = Run the tests
+require_locked_deps = true
 deps =
-    requests >=2.19,<3.0
-    toml == 0.10.0
-    pytest >=5.4
-
-# ...becomes this:
-[testenv]
-description = My cool test environment
-deps =
-    requests
-    toml
     pytest
+    pytest-cov
+    black
+    pylint
+    mypy
+commands = ...
 ```
 
-3. Run Tox with the `--recreate` flag to rebuild the test environments:
+To require specific dependencies be installed from the Poetry lockfile, and let the rest be
+installed using the default Tox installation backend, add the suffix `@poetry` to the dependencies.
+In the example below the `pytest`, `pytest-cov`, and `black` dependencies will be installed from
+the lockfile while `pylint` and `mypy` will be installed using the versions specified here:
 
-```
+```ini
-poetry run tox --recreate
+[testenv]
+description = Run the tests
+require_locked_deps = true
+deps =
+    pytest@poetry
+    pytest-cov@poetry
+    black@poetry
+    pylint >=2.5.0
+    mypy == 0.770
+commands = ...
 ```
 
-4. 💸 Profit 💸
+**Note:** Regardless of the settings outlined above, all dependencies of the project package (the
+one Tox is testing) will always be installed from the lockfile.
 
 
-## Limitations
+## Usage Examples
 
-* In general, any command line or INI settings that affect how Tox installs environment
+After installing the plugin to a project your Tox automation is already benefiting from the
-  dependencies will be disabled by installing this plugin. A non-exhaustive and untested
+lockfile: when Tox installs your project package to one of your environments, all the dependencies
-  list of the INI options that are not expected to work with this plugin is below:
+of your project package will be installed using the versions specified in the lockfile. This
+happens automatically and requires no configuration changes.
+
+But what about the rest of your Tox environment dependencies?
+
+Let's use an example `tox.ini` file, below, that defines two environments: the main `testenv` for
+running the project tests and `testenv:check` for running some other helpful tools:
+
+```ini
+[tox]
+envlist = py37, static
+isolated_build = true
+
+[testenv]
+description = Run the tests
+deps =
+    pytest == 5.3.0
+commands = ...
+
+[testenv:check]
+description = Static formatting and quality enforcement
+deps =
+    pylint >=2.4.4,<2.6.0
+    mypy == 0.770
+    black --pre
+commands = ...
+```
+
+Let's focus on the `testenv:check` environment first. In this project there's no reason that any
+of these tools should be a different version than what a human developer is using when installing
+from the lockfile. We can require that these dependencies be installed from the lockfile by adding
+the option `require_locked_deps = true` to the environment config, but this will cause an error:
+
+```ini
+[testenv:check]
+description = Static formatting and quality enforcement
+require_locked_deps = true
+deps =
+    pylint >=2.4.4,<2.6.0
+    mypy == 0.770
+    black --pre
+commands = ...
+```
+
+Running Tox using this config gives us this error:
+
+```
+tox_poetry_installer.LockedDepVersionConflictError: Locked dependency 'pylint >=2.4.4,<2.6.0' cannot include version specifier
+```
+
+This is because we told the Tox environment to require all dependencies be locked, but then also
+specified a specific version constraint for Pylint. With the `require_locked_deps = true` setting
+Tox expects all dependencies to take their version from the lockfile, so when it gets conflicting
+information it errors. We can fix this by simply removing all version specifiers from the
+environment dependency list:
+
+```ini
+[testenv:check]
+description = Static formatting and quality enforcement
+require_locked_deps = true
+deps =
+    pylint
+    mypy
+    black
+commands = ...
+```
+
+Now all the dependencies will be installed from the lockfile. If Poetry updates the lockfile with
+a new version then that updated version will be automatically installed when the Tox environment is
+recreated.
+
+Now let's look at the `testenv` environment. Let's make the same changes to the `testenv`
+environment that we made to `testenv:check` above; remove the PyTest version and add
+`require_locked_deps = true`. Then imagine that we want to add the
+[Requests](https://requests.readthedocs.io/en/master/) library to the test environment: we
+can add `requests` as a dependency of the test environment, but this will cause an error:
+
+```ini
+[testenv]
+description = Run the tests
+require_locked_deps = true
+deps =
+    pytest
+    requests
+commands = ...
+```
+
+Running Tox with this config gives us this error:
+
+```
+tox_poetry_installer.LockedDepNotFoundError: No version of locked dependency 'requests' found in the project lockfile
+```
+
+This is because `requests` is not in our lockfile yet. Tox will refuse to install a dependency
+that isn't in the lockfile to an an environment that specifies `require_locked_deps = true`. We
+can fix this by running `poetry add requests --dev` to add it to the lockfile.
+
+Now let's combine dependencies from the lockfile with dependencies that are
+specified in-line in the Tox environment configuration.
+[This isn't generally recommended](#why-would-i-use-this), but it is a valid use case and
+fully supported by this plugin. Let's modify the `testenv` configuration to install PyTest
+from the lockfile but then install an older version of the Requests library.
+
+The first thing to do is remove the `require_locked_deps = true` setting so that we can install
+Requests as an unlocked dependency. Then we can add our version specifier to the `requests`
+entry in the dependency list:
+
+```ini
+[testenv]
+description = Run the tests
+deps =
+    pytest
+    requests >=2.2.0,<2.10.0
+commands = ...
+```
+
+However we still want `pytest` to be installed from the lockfile, so the final step is to tell Tox
+to install it from the lockfile by adding the suffix `@poetry` to the `pytest` entry in the
+dependency list:
+
+```ini
+[testenv]
+description = Run the tests
+deps =
+    pytest@poetry
+    requests >=2.2.0,<2.10.0
+commands = ...
+```
+
+Now when the `testenv` environment is created it will install PyTest (and all of its dependencies)
+from the lockfile while it will install Requests (and all of its dependencies) using the default
+Tox installation backend.
+
+
+## Known Drawbacks and Problems
+
+* The following `tox.ini` configuration options have no effect on the dependencies installed from
+  the Poetry lockfile (note that they will still affect unlocked dependencies):
   * [`install_command`](https://tox.readthedocs.io/en/latest/config.html#conf-install_command)
   * [`pip_pre`](https://tox.readthedocs.io/en/latest/config.html#conf-pip_pre)
   * [`downloadcache`](https://tox.readthedocs.io/en/latest/config.html#conf-downloadcache) (deprecated)
   * [`download`](https://tox.readthedocs.io/en/latest/config.html#conf-download)
   * [`indexserver`](https://tox.readthedocs.io/en/latest/config.html#conf-indexserver)
   * [`usedevelop`](https://tox.readthedocs.io/en/latest/config.html#conf-indexserver)
-  * [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
 
-* When the plugin is enabled all dependencies for all environments will use the Poetry backend
+* Tox environments automatically inherit their settings from the main `testenv` environment. This
-  provided by the plugin; this functionality cannot be disabled on a per-environment basis.
+  means that if the `require_locked_deps = true` is specified for the `testenv` environment then
+  all environments will also require locked dependencies. This can be overwritten by explicitly
+  specifying `require_locked_deps = false` on child environments where unlocked dependencies are
+  needed.
 
-* Alternative versions cannot be specified alongside versions from the lockfile. All
+* There are a handful of packages that cannot be installed from the lockfile, whether as specific
-  dependencies are installed from the lockfile and alternative versions cannot be specified
+  dependencies or as transient dependencies (dependencies of dependencies). This is due to
-  in the Tox configuration.
+  [an ongoing discussion in the Poetry project](https://github.com/python-poetry/poetry/issues/1584);
+  the list of dependencies that cannot be installed from the lockfile can be found
+  [here](https://github.com/python-poetry/poetry/blob/cc8f59a31567f806be868aba880ae0642d49b74e/poetry/puzzle/provider.py#L55).
+  This plugin will skip these dependencies entirely, but log a warning when they are encountered.
 
 
 ## Why would I use this?
 
 **Introduction**
 
-The lockfile is a file generated by a package manager for a project that lists what
+The lockfile is a file generated by a package manager for a project that records what
-dependencies are installed, the versions of those dependencies, and additional metadata that
+dependencies are installed, the versions of those dependencies, and any additional metadata that
-the package manager can use to recreate the local project environment. This allows developers
+the package manager needs to recreate the local project environment. This allows developers
 to have confidence that a bug they are encountering that may be caused by one of their
 dependencies will be reproducible on another device. In addition, installing a project
 environment from a lockfile gives confidence that automated systems running tests or performing
-builds are using the same environment that a developer is.
+builds are using the same environment as a developer.
 
 [Poetry](https://python-poetry.org/) is a project dependency manager for Python projects, and
-as such it creates and manages a lockfile so that its users can benefit from all the features
+so it creates and manages a lockfile so that its users can benefit from all the features
 described above. [Tox](https://tox.readthedocs.io/en/latest/#what-is-tox) is an automation tool
 that allows Python developers to run tests suites, perform builds, and automate tasks within
-self contained [Python virtual environments](https://docs.python.org/3/tutorial/venv.html).
+self-contained [Python virtual environments](https://docs.python.org/3/tutorial/venv.html).
-To make these environments useful, Tox supports installing per-environment dependencies.
+To make these environments useful Tox supports installing dependencies in each environment.
 However, since these environments are created on the fly and Tox does not maintain a lockfile,
 there can be subtle differences between the dependencies a developer is using and the
 dependencies Tox uses.
@@ -109,19 +288,17 @@ dependencies Tox uses.
 This is where this plugin comes into play.
 
 By default Tox uses [Pip](https://docs.python.org/3/tutorial/venv.html) to install the
-PEP-508 compliant dependencies to a test environment. A more robust way to do this is to
+PEP-508 compliant dependencies to a test environment. This plugin extends the default
-install dependencies directly from the lockfile so that the version installed to the Tox
+Tox dependency installation behavior to support installing dependencies using a Poetry-based
-environment always matches the version Poetry specifies. This plugin overwrites the default
+installation method that makes use of the dependency metadata from Poetry's lockfile.
-Tox dependency installation behavior and replaces it with a Poetry-based installation using
-the dependency metadata from the lockfile.
 
 **The Problem**
 
-Environment dependencies for a Tox environment are usually done in PEP-508 format like the
+Environment dependencies for a Tox environment are usually specified in PEP-508 format, like
-below example
+the below example:
 
 ```ini
-# tox.ini
+# from tox.ini
 ...
 
 [testenv]
@@ -134,31 +311,35 @@ deps =
 ...
 ```
 
-Perhaps these dependencies are also useful during development, so they can be added to the
+Let's assume these dependencies are also useful during development, so they can be added to the
 Poetry environment using this command:
 
  ```
- poetry add foo==1.2.3 bar>=1.3,<2.0 baz --dev
+ poetry add --dev \
+    foo==1.2.3 \
+    bar>=1.3,<2.0 \
+    baz
  ```
 
- However there are three potential problems that could arise from each of these environment
+ However there is a potential problem that could arise from each of these environment
  dependencies that would _only_ appear in the Tox environment and not in the Poetry
- environment:
+ environment in use by a developer:
 
  * **The `foo` dependency is pinned to a specific version:** let's imagine a security
    vulnerability is discovered in `foo` and the maintainers release version `1.2.4` to fix
-   it. A developer can run `poetry remove foo && poetry add foo^1.2` to get the new version,
+   it. A developer can run `poetry remove foo` and then `poetry add foo^1.2` to get the new
-   but the Tox environment is left unchanged. The developer environment specified by the
+   version, but the Tox environment is left unchanged. The development environment, as defined by
-   lockfile is now patched against the vulnerability, but the Tox environment is not.
+   the lockfile, is now patched against the vulnerability but the Tox environment is not.
 
 * **The `bar` dependency specifies a dynamic range:** a dynamic range allows a range of
   versions to be installed, but the lockfile will have an exact version specified so that
   the Poetry environment is reproducible; this allows versions to be updated with
-  `poetry update` rather than with the `remove` and `add` used above. If the maintainers of
+  `poetry update` rather than with the `remove` and `add` commands used above. If the
-  `bar` release version `1.6.0` then the Tox environment will install it because it is valid
+  maintainers of `bar` release version `1.6.0` then the Tox environment will install it
-  for the specified version range, meanwhile the Poetry environment will continue to install
+  because it is valid for the specified version range. Meanwhile the Poetry environment will
-  the version from the lockfile until `poetry update bar` explicitly updates it. The
+  continue to install the version from the lockfile until `poetry update bar` explicitly
-  development environment is now has a different version of `bar` than the Tox environment.
+  updates it. The development environment is now has a different version of `bar` than the Tox
+  environment.
 
 * **The `baz` dependency is unpinned:** unpinned dependencies are
   [generally a bad idea](https://python-poetry.org/docs/faq/#why-are-unbound-version-constraints-a-bad-idea),
@@ -167,27 +348,26 @@ Poetry environment using this command:
   but Pip (via Tox) will interpret it as a wildcard. If the latest version of `baz` is `1.0.0`
   then `poetry add baz` will result in a constraint of `baz>=1.0.0,<2.0.0` while the Tox
   environment will have a constraint of `baz==*`. The Tox environment can now install an
-  incompatible version of `baz` that cannot be easily caught using `poetry update`.
+  incompatible version of `baz` and any errors that causes cannot be replicated using `poetry update`.
 
 All of these problems can apply not only to the dependencies specified for a Tox environment,
-but also to the dependencies of those dependencies, and so on.
+but also to the dependencies of those dependencies, those dependencies' dependencies, and so on.
 
 **The Solution**
 
-This plugin requires that all dependencies specified for all Tox environments be unbound
+This plugin allows dependencies specified in Tox environment take their version directly from
-with no version constraint specified at all. This seems counter-intuitive given the problems
+the Poetry lockfile without needing an independent version to be specified in the Tox
-outlined above, but what it allows the plugin to do is offload all version management to
+environment configuration. The modified version of the example environment given below appears
-Poetry.
+less stable than the one presented above because it does not specify any versions for its
-
+dependencies:
-On initial inspection, the environment below appears less stable than the one presented above
-because it does not specify any versions for its dependencies:
 
 ```ini
-# tox.ini
+# from tox.ini
 ...
 
 [testenv]
 description = Some very cool tests
+require_locked_deps = true
 deps =
     foo
     bar
@@ -196,19 +376,17 @@ deps =
 ...
 ```
 
-However with the `tox-poetry-installer` plugin installed this instructs Tox to install these
+However with the `tox-poetry-installer` plugin installed the `require_locked_deps = true`
-dependencies using the Poetry lockfile so that the version installed to the Tox environment
+setting means that Tox will install these dependencies from the Poetry lockfile so that the
-exactly matches the version Poetry is managing. When `poetry update` updates the lockfile
+version installed to the Tox environment exactly matches the version Poetry is managing. When
-with new dependency versions, Tox will automatically install these new versions without needing
+`poetry update` updates the lockfile with new versions of these dependencies, Tox will
-any changes to the configuration.
+automatically install these new versions without needing any changes to the configuration.
-
-All dependencies are specified in one place (the lockfile) and dependency version management is
-handled by a tool dedicated to that task (Poetry).
 
 
 ## Developing
 
-This project requires Poetry-1.0+, see the [installation instructions here](https://python-poetry.org/docs/#installation).
+This project requires a developer to have Poetry version 1.0+ installed on their workstation, see
+the [installation instructions here](https://python-poetry.org/docs/#installation).
 
 ```bash
 # Clone the repository...
@@ -234,6 +412,10 @@ poetry run tox
 All project contributors and participants are expected to adhere to the
 [Contributor Covenant Code of Conduct, Version 2](CODE_OF_CONDUCT.md).
 
+The `devel` branch has the latest (potentially unstable) changes. The
+[tagged versions](https://github.com/enpaul/tox-poetry-installer/releases) correspond to the
+releases on PyPI.
+
 * To report a bug, request a feature, or ask for assistance, please
   [open an issue on the Github repository](https://github.com/enpaul/tox-poetry-installer/issues/new).
 * To report a security concern or code of conduct violation, please contact the project author
@@ -247,7 +429,7 @@ All project contributors and participants are expected to adhere to the
 ## Roadmap
 
 This project is under active development and is classified as alpha software, not yet ready
-usage in production systems.
+for usage in production environments.
 
 * Beta classification will be assigned when the initial feature set is finalized
 * Stable classification will be assigned when the test suite covers an acceptable number of
@@ -257,19 +439,21 @@ usage in production systems.
 
 - [X] Verify that primary package dependencies (from the `.package` env) are installed
       correctly using the Poetry backend.
-- [ ] Support the [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
+- [X] Support the [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
-      Tox configuration option
+      Tox configuration option ([#4](https://github.com/enpaul/tox-poetry-installer/issues/4))
-- [ ] Add per-environment Tox configuration option to fall back to default installation
+- [X] Add per-environment Tox configuration option to fall back to default installation
       backend.
-- [ ] Add detection of a changed lockfile to automatically trigger a rebuild of Tox
-      environments when necessary.
 - [ ] Add warnings when an unsupported Tox configuration option is detected while using the
-      Poetry backend.
+      Poetry backend. ([#5](https://github.com/enpaul/tox-poetry-installer/issues/5))
 - [X] Add trivial tests to ensure the project metadata is consistent between the pyproject.toml
       and the module constants.
-- [ ] Update to use [poetry-core](https://github.com/python-poetry/poetry-core)
+- [X] Update to use [poetry-core](https://github.com/python-poetry/poetry-core) and
-      Tox configuration option) and improve robustness of the Tox and Poetry module imports
+      improve robustness of the Tox and Poetry module imports
-      to avoid potentially breaking API changes in upstream packages.
+      to avoid potentially breaking API changes in upstream packages. ([#2](https://github.com/enpaul/tox-poetry-installer/issues/2))
+- [ ] Find and implement a way to mitigate the [UNSAFE_DEPENDENCIES issue](https://github.com/python-poetry/poetry/issues/1584) in Poetry.
+      ([#6](https://github.com/enpaul/tox-poetry-installer/issues/6))
+- [ ] Fix logging to make proper use of Tox's logging reporter infrastructure ([#3](https://github.com/enpaul/tox-poetry-installer/issues/3))
+- [ ] Add configuration option for installing all dev-dependencies to a testenv ([#14](https://github.com/enpaul/tox-poetry-installer/issues/14))
 
 ### Path to Stable
 
@@ -277,6 +461,6 @@ Everything in Beta plus...
 
 - [ ] Add tests for each feature version of Tox between 2.3 and 3.20
 - [ ] Add tests for Python-3.6, 3.7, and 3.8
-- [ ] Add Github Actions based CI
+- [X] Add Github Actions based CI
 - [ ] Add CI for CPython, PyPy, and Conda
 - [ ] Add CI for Linux and Windows

pyproject.toml

@@ -1,11 +1,14 @@
 [tool.poetry]
 name = "tox-poetry-installer"
-version = "0.1.3"
+version = "0.3.1"
 license = "MIT"
 authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 description = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
 repository = "https://github.com/enpaul/tox-poetry-installer/"
-packages = [{include = "tox_poetry_installer.py"}]
+packages = [
+  {include = "tox_poetry_installer.py"},
+  {include = "tests/*.py", format = "sdist"}
+]
 keywords = ["tox", "poetry", "plugin"]
 readme = "README.md"
 classifiers = [
@@ -27,7 +30,8 @@ poetry_installer = "tox_poetry_installer"
 
 [tool.poetry.dependencies]
 python = "^3.6"
-poetry = ">=1.0.0, <1.1.0"
+poetry = "^1.0.0"
+poetry-core = "^1.0.0"
 tox = "^2.3.0 || ^3.0.0"
 
 [tool.poetry.dev-dependencies]

tox.ini

@@ -4,6 +4,7 @@ isolated_build = true
 
 [testenv]
 description = Run the tests
+require_locked_deps = true
 deps =
     pytest
     pytest-cov
@@ -13,6 +14,7 @@ commands =
 
 [testenv:static]
 description = Static formatting and quality enforcement
+require_locked_deps = true
 basepython = python3.8
 ignore_errors = true
 deps =
@@ -30,6 +32,7 @@ commands =
 
 [testenv:static-tests]
 description = Static formatting and quality enforcement for the tests
+require_locked_deps = true
 basepython = python3.8
 ingore_errors = true
 deps =
@@ -42,11 +45,12 @@ allowlist_externals =
 commands =
     black {toxinidir}/tests/
     bash -c "reorder-python-imports {toxinidir}/tests/*.py --unclassifiable-application-module tox_poetry_installer"
-    pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tests/
+    bash -c "pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tests/*.py"
-    mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tests/
+    bash -c "mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tests/*.py"
 
 [testenv:security]
 description = Security checks
+require_locked_deps = true
 basepython = python3.8
 ignore_errors = true
 skip_install = true

tox_poetry_installer.py

@@ -1,49 +1,146 @@
 """Tox plugin for installing environments using Poetry
 
-This plugin makes use of the ``tox_testenv_install_deps`` Tox plugin hook to replace the default
+This plugin makes use of the ``tox_testenv_install_deps`` Tox plugin hook to augment the default
 installation functionality to install dependencies from the Poetry lockfile for the project. It
 does this by using ``poetry`` to read in the lockfile, identify necessary dependencies, and then
 use Poetry's ``PipInstaller`` class to install those packages into the Tox environment.
+
+Quick definition of terminology:
+
+* "project package" - the package that Tox is testing, usually the one the current project is
+  is developing; definitionally, this is the package that is built by Tox in the ``.package`` env.
+* "project package dependency" or "project dependency" - a dependency required by the project
+  package for installation; i.e. a package that would be installed when running
+  ``pip install <project package>``.
+* "environment dependency" - a dependency specified for a given testenv in the Tox configuration.
+* "locked dependency" - a package that is present in the Poetry lockfile and will be installed
+  according to the metadata in the lockfile.
+* "unlocked dependency" - a package that is either not present in the Poetry lockfile or is not
+  specified to be installed according to the metadata in the lockfile.
+* "transiety dependency" - a package not explicitly specified for installation, but required by a
+  package that is explicitly specified.
 """
 from pathlib import Path
 from typing import Dict
 from typing import List
-from typing import Optional
+from typing import NamedTuple
 from typing import Sequence
+from typing import Set
 from typing import Tuple
 
+from poetry.core.packages import Package as PoetryPackage
 from poetry.factory import Factory as PoetryFactory
-from poetry.factory import Poetry
 from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
 from poetry.io.null_io import NullIO as PoetryNullIO
-from poetry.packages import Package as PoetryPackage
+from poetry.poetry import Poetry
 from poetry.puzzle.provider import Provider as PoetryProvider
 from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
 from tox import hookimpl
 from tox import reporter
 from tox.action import Action as ToxAction
+from tox.config import DepConfig as ToxDepConfig
+from tox.config import Parser as ToxParser
 from tox.venv import VirtualEnv as ToxVirtualEnv
 
 
 __title__ = "tox-poetry-installer"
 __summary__ = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
-__version__ = "0.1.3"
+__version__ = "0.3.1"
 __url__ = "https://github.com/enpaul/tox-poetry-installer/"
 __license__ = "MIT"
 __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 
 
+# Valid PEP508 version delimiters. These are used to test whether a given string (specifically a
+# dependency name) is just a package name or also includes a version identifier.
 _PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<")
 
+# Prefix all reporter messages should include to indicate that they came from this module in the
+# console output.
 _REPORTER_PREFIX = f"[{__title__}]:"
 
+# Suffix that indicates an env dependency should be treated as a locked dependency and thus be
+# installed from the lockfile. Will be automatically stripped off of a dependency name during
+# sorting so that the resulting string is just the valid package name. This becomes optional when
+# the "require_locked_deps" option is true for an environment; in that case a bare dependency like
+# 'foo' is treated the same as an explicitly locked dependency like 'foo@poetry'
+_MAGIC_SUFFIX_MARKER = "@poetry"
+
+
+# Map of package names to the package object
+PackageMap = Dict[str, PoetryPackage]
+
+
+class _SortedEnvDeps(NamedTuple):
+    unlocked_deps: List[ToxDepConfig]
+    locked_deps: List[ToxDepConfig]
+
 
 class ToxPoetryInstallerException(Exception):
     """Error while installing locked dependencies to the test environment"""
 
 
-class NoLockedDependencyError(ToxPoetryInstallerException):
+class LockedDepVersionConflictError(ToxPoetryInstallerException):
-    """Cannot install a package that is not in the lockfile"""
+    """Locked dependencies cannot specify an alternate version for installation"""
+
+
+class LockedDepNotFoundError(ToxPoetryInstallerException):
+    """Locked dependency was not found in the lockfile"""
+
+
+class ExtraNotFoundError(ToxPoetryInstallerException):
+    """Project package extra not defined in project's pyproject.toml"""
+
+
+def _sort_env_deps(venv: ToxVirtualEnv) -> _SortedEnvDeps:
+    """Sorts the environment dependencies by lock status
+
+    Lock status determines whether a given environment dependency will be installed from the
+    lockfile using the Poetry backend, or whether this plugin will skip it and allow it to be
+    installed using the default pip-based backend (an unlocked dependency).
+
+    .. note:: A locked dependency must follow a required format. To avoid reinventing the wheel
+              (no pun intended) this module does not have any infrastructure for parsing PEP-508
+              version specifiers, and so requires locked dependencies to be specified with no
+              version (the installed version being taken from the lockfile). If a dependency is
+              specified as locked and its name is also a PEP-508 string then an error will be
+              raised.
+    """
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} sorting {len(venv.envconfig.deps)} env dependencies by lock requirement"
+    )
+    unlocked_deps = []
+    locked_deps = []
+
+    for dep in venv.envconfig.deps:
+        if venv.envconfig.require_locked_deps:
+            reporter.verbosity1(
+                f"{_REPORTER_PREFIX} lock required for env, treating '{dep.name}' as locked env dependency"
+            )
+            dep.name = dep.name.replace(_MAGIC_SUFFIX_MARKER, "")
+            locked_deps.append(dep)
+        else:
+            if dep.name.endswith(_MAGIC_SUFFIX_MARKER):
+                reporter.verbosity1(
+                    f"{_REPORTER_PREFIX} specification includes marker '{_MAGIC_SUFFIX_MARKER}', treating '{dep.name}' as locked env dependency"
+                )
+                dep.name = dep.name.replace(_MAGIC_SUFFIX_MARKER, "")
+                locked_deps.append(dep)
+            else:
+                reporter.verbosity1(
+                    f"{_REPORTER_PREFIX} specification does not include marker '{_MAGIC_SUFFIX_MARKER}', treating '{dep.name}' as unlocked env dependency"
+                )
+                unlocked_deps.append(dep)
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} identified {len(locked_deps)} locked env dependencies: {[item.name for item in locked_deps]}"
+    )
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} identified {len(unlocked_deps)} unlocked env dependencies: {[item.name for item in unlocked_deps]}"
+    )
+
+    return _SortedEnvDeps(locked_deps=locked_deps, unlocked_deps=unlocked_deps)
 
 
 def _install_to_venv(
@@ -55,6 +152,11 @@ def _install_to_venv(
     :param venv: Tox virtual environment to install the packages to
     :param packages: List of packages to install to the virtual environment
     """
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} Installing {len(packages)} packages to environment at {venv.envconfig.envdir}"
+    )
+
     installer = PoetryPipInstaller(
         env=PoetryVirtualEnv(path=Path(venv.envconfig.envdir)),
         io=PoetryNullIO(),
@@ -66,9 +168,7 @@ def _install_to_venv(
         installer.install(dependency)
 
 
-def _find_locked_dependencies(
+def _find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPackage]:
-    poetry: Poetry, dependency_name: str
-) -> List[PoetryPackage]:
     """Using a poetry object identify all dependencies of a specific dependency
 
     :param poetry: Populated poetry object which can be used to build a populated locked
@@ -80,46 +180,142 @@ def _find_locked_dependencies(
     .. note:: The package corresponding to the dependency named by ``dependency_name`` is included
               in the list of returned packages.
     """
-    packages: Dict[str, PoetryPackage] = {
-        package.name: package
-        for package in poetry.locker.locked_repository(True).packages
-    }
 
     try:
         top_level = packages[dependency_name]
 
-        def find_transients(name: str) -> List[PoetryPackage]:
+        def find_deps_of_deps(name: str) -> List[PoetryPackage]:
             if name in PoetryProvider.UNSAFE_PACKAGES:
                 reporter.warning(
-                    f"{_REPORTER_PREFIX} installing '{name}' using Poetry is not supported; skipping"
+                    f"{_REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
                 )
                 return []
             transients = [packages[name]]
             for dep in packages[name].requires:
-                transients += find_transients(dep.name)
+                transients += find_deps_of_deps(dep.name)
             return transients
 
-        return find_transients(top_level.name)
+        return set(find_deps_of_deps(top_level.name))
 
     except KeyError:
         if any(
             delimiter in dependency_name for delimiter in _PEP508_VERSION_DELIMITERS
         ):
-            message = "specifying a version in the tox environment definition is incompatible with installing from a lockfile"
+            raise LockedDepVersionConflictError(
-        else:
+                f"Locked dependency '{dependency_name}' cannot include version specifier"
-            message = (
+            ) from None
-                "no version of the package was found in the current project's lockfile"
+        raise LockedDepNotFoundError(
-            )
+            f"No version of locked dependency '{dependency_name}' found in the project lockfile"
-
-        raise NoLockedDependencyError(
-            f"Cannot install requirement '{dependency_name}': {message}"
         ) from None
 
 
+def _install_env_dependencies(
+    venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
+):
+    """Install the packages for a specified testenv
+
+    Processes the tox environment config, identifies any locked environment dependencies, pulls
+    them from the lockfile, and installs them to the virtual environment.
+
+    :param venv: Tox virtual environment to install the packages to
+    :param poetry: Poetry object the packages were sourced from
+    :param packages: Mapping of package names to the corresponding package object
+    """
+    env_deps = _sort_env_deps(venv)
+
+    dependencies: List[PoetryPackage] = []
+    for dep in env_deps.locked_deps:
+        try:
+            dependencies += _find_transients(packages, dep.name.lower())
+        except ToxPoetryInstallerException as err:
+            venv.status = "lockfile installation failed"
+            reporter.error(f"{_REPORTER_PREFIX} {err}")
+            raise err
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(env_deps.locked_deps)} locked env dependencies"
+    )
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} updating env config with {len(env_deps.unlocked_deps)} unlocked env dependencies for installation using the default backend"
+    )
+    venv.envconfig.deps = env_deps.unlocked_deps
+
+    reporter.verbosity0(
+        f"{_REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile"
+    )
+    _install_to_venv(poetry, venv, dependencies)
+
+
+def _install_project_dependencies(
+    venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
+):
+    """Install the dependencies of the project package
+
+    Install all primary dependencies of the project package.
+
+    :param venv: Tox virtual environment to install the packages to
+    :param poetry: Poetry object the packages were sourced from
+    :param packages: Mapping of package names to the corresponding package object
+    """
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} performing installation of project dependencies"
+    )
+
+    base_dependencies: List[PoetryPackage] = [
+        packages[item.name]
+        for item in poetry.package.requires
+        if not item.is_optional()
+    ]
+
+    extra_dependencies: List[PoetryPackage] = []
+    for extra in venv.envconfig.extras:
+        try:
+            extra_dependencies += [
+                packages[item.name] for item in poetry.package.extras[extra]
+            ]
+        except KeyError:
+            raise ExtraNotFoundError(
+                f"Environment '{venv.name}' specifies project extra '{extra}' which was not found in the lockfile"
+            ) from None
+
+    dependencies: List[PoetryPackage] = []
+    for dep in base_dependencies + extra_dependencies:
+        try:
+            dependencies += _find_transients(packages, dep.name.lower())
+        except ToxPoetryInstallerException as err:
+            venv.status = "lockfile installation failed"
+            reporter.error(f"{_REPORTER_PREFIX} {err}")
+            raise err
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies"
+    )
+
+    reporter.verbosity0(
+        f"{_REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile"
+    )
+    _install_to_venv(poetry, venv, dependencies)
+
+
 @hookimpl
-def tox_testenv_install_deps(
+def tox_addoption(parser: ToxParser):
-    venv: ToxVirtualEnv, action: ToxAction
+    """Add required configuration options to the tox INI file
-) -> Optional[List[PoetryPackage]]:
+
+    Adds the ``require_locked_deps`` configuration option to the venv to check whether all
+    dependencies should be treated as locked or not.
+    """
+
+    parser.add_testenv_attribute(
+        name="require_locked_deps",
+        type="bool",
+        default=False,
+        help="Require all dependencies in the environment be installed using the Poetry lockfile",
+    )
+
+
+@hookimpl
+def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction):
     """Install the dependencies for the current environment
 
     Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
@@ -131,47 +327,51 @@ def tox_testenv_install_deps(
     """
 
     if action.name == venv.envconfig.config.isolated_build_env:
+        # Skip running the plugin for the packaging environment. PEP-517 front ends can handle
+        # that better than we can, so let them do their thing. More to the point: if you're having
+        # problems in the packaging env that this plugin would solve, god help you.
         reporter.verbosity1(
             f"{_REPORTER_PREFIX} skipping isolated build env '{action.name}'"
         )
-        return None
+        return
 
-    poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
+    try:
+        poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
+    except RuntimeError:
+        # Support running the plugin when the current tox project does not use Poetry for its
+        # environment/dependency management.
+        #
+        # ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
+        # is) raised in many different places for different purposes.
+        reporter.verbosity1(
+            f"{_REPORTER_PREFIX} project does not use Poetry for env management, skipping installation of locked dependencies"
+        )
+        return
 
     reporter.verbosity1(
         f"{_REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}"
     )
 
-    dependencies: List[PoetryPackage] = []
+    package_map: PackageMap = {
-    for env_dependency in venv.envconfig.deps:
+        package.name: package
-        dependencies += _find_locked_dependencies(poetry, env_dependency.name)
+        for package in poetry.locker.locked_repository(True).packages
+    }
 
-    reporter.verbosity1(
+    # Handle the installation of any locked env dependencies from the lockfile
-        f"{_REPORTER_PREFIX} identified {len(dependencies)} actual dependencies from {len(venv.envconfig.deps)} specified env dependencies"
+    _install_env_dependencies(venv, poetry, package_map)
-    )
 
-    reporter.verbosity0(
+    # Handle the installation of the package dependencies from the lockfile if the package is
-        f"{_REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile"
+    # being installed to this venv; otherwise skip installing the package dependencies
-    )
+    if venv.envconfig.skip_install:
-    _install_to_venv(poetry, venv, dependencies)
-
-    if not venv.envconfig.skip_install:
         reporter.verbosity1(
-            f"{_REPORTER_PREFIX} env specifies 'skip_install = false', performing installation of dev-package dependencies"
+            f"{_REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package"
         )
+        return
 
-        primary_dependencies = poetry.locker.locked_repository(False).packages
+    if venv.envconfig.config.skipsdist:
         reporter.verbosity1(
-            f"{_REPORTER_PREFIX} identified {len(primary_dependencies)} dependencies of dev-package"
+            f"{_REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package"
         )
+        return
 
-        reporter.verbosity0(
+    _install_project_dependencies(venv, poetry, package_map)
-            f"{_REPORTER_PREFIX} ({venv.name}) installing {len(primary_dependencies)} dev-package dependencies from lockfile"
-        )
-        _install_to_venv(poetry, venv, primary_dependencies)
-    else:
-        reporter.verbosity1(
-            f"{_REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of dev-package package"
-        )
-
-    return dependencies