Merge pull request #16 from enpaul/enp/fix

Fix support for extras installation

.github/workflows/ci.yaml

@@ -0,0 +1,41 @@
+---
+name: CI
+on:
+  pull_request:
+    types: ["opened", "synchronize"]
+  push:
+    branches: ["devel"]
+jobs:
+  Test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python:
+          - version: 3.6
+            toxenv: py36
+          - version: 3.7
+            toxenv: py37
+          - version: 3.8
+            toxenv: py38
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python.version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python.version }}
+      - name: Install tox
+        run: pip install "tox>=3.20.0,<3.21.0" --upgrade
+      - name: Run tests via ${{ matrix.python.toxenv }}
+        run: tox -e ${{ matrix.python.toxenv }}
+  Check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install tox requirements
+        run: pip install "tox>=3.20.0,<3.21.0" --upgrade
+      - name: Run meta checks
+        run: tox -e static -e static-tests -e security

.pre-commit-config.yaml

@@ -4,14 +4,14 @@ repos:
     rev: 20.8b1
     hooks:
       - id: black
-        language_version: python3.7
+        language_version: python3
 
   - repo: https://github.com/asottile/blacken-docs
     rev: v0.5.0
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==20.8b1]
-        language_version: python3.7
+        language_version: python3
 
   - repo: https://github.com/asottile/reorder_python_imports
     rev: v2.3.5

README.md

@@ -5,9 +5,10 @@ dependencies to be installed using [Poetry](https://python-poetry.org/) from its
 
 ⚠️ **This project is alpha software and should not be used in production environments** ⚠️
 
-[![image](https://img.shields.io/pypi/l/tox-poetry-installer)](https://opensource.org/licenses/MIT)
-[![image](https://img.shields.io/pypi/v/tox-poetry-installer)](https://pypi.org/project/tox-poetry-installer/)
-[![image](https://img.shields.io/pypi/pyversions/tox-poetry-installer)](https://www.python.org)
+[![ci-status](https://github.com/enpaul/tox-poetry-installer/workflows/CI/badge.svg?event=push)](https://github.com/enpaul/tox-poetry-installer/actions)
+[![license](https://img.shields.io/pypi/l/tox-poetry-installer)](https://opensource.org/licenses/MIT)
+[![pypi-version](https://img.shields.io/pypi/v/tox-poetry-installer)](https://pypi.org/project/tox-poetry-installer/)
+[![python-versions](https://img.shields.io/pypi/pyversions/tox-poetry-installer)](https://www.python.org)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 **Documentation**
@@ -26,7 +27,9 @@ dependencies to be installed using [Poetry](https://python-poetry.org/) from its
 Related resources:
 * [Poetry Python Project Manager](https://python-poetry.org/)
 * [Tox Automation Project](https://tox.readthedocs.io/en/latest/)
-* [Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html)
+* [Poetry Dev-Dependencies Tox Plugin](https://github.com/sinoroc/tox-poetry-dev-dependencies)
+* [Poetry Tox Plugin](https://github.com/tkukushkin/tox-poetry)
+* [Other Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html)
 
 
 ## Installation
@@ -246,13 +249,6 @@ Tox installation backend.
   * [`indexserver`](https://tox.readthedocs.io/en/latest/config.html#conf-indexserver)
   * [`usedevelop`](https://tox.readthedocs.io/en/latest/config.html#conf-indexserver)
 
-* The [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras) setting in `tox.ini`
-  does not work. Optional dependencies of the project package will not be installed to Tox
-  environments. (See the [road map](#roadmap))
-
-* The plugin currently depends on `poetry<1.1.0`. This can be a different version than Poetry being
-  used for actual project development. (See the [road map](#roadmap))
-
 * Tox environments automatically inherit their settings from the main `testenv` environment. This
   means that if the `require_locked_deps = true` is specified for the `testenv` environment then
   all environments will also require locked dependencies. This can be overwritten by explicitly
@@ -389,7 +385,8 @@ automatically install these new versions without needing any changes to the conf
 
 ## Developing
 
-This project requires Poetry version 1.0+, see the [installation instructions here](https://python-poetry.org/docs/#installation).
+This project requires a developer to have Poetry version 1.0+ installed on their workstation, see
+the [installation instructions here](https://python-poetry.org/docs/#installation).
 
 ```bash
 # Clone the repository...
@@ -442,7 +439,7 @@ for usage in production environments.
 
 - [X] Verify that primary package dependencies (from the `.package` env) are installed
       correctly using the Poetry backend.
-- [ ] Support the [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
+- [X] Support the [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
       Tox configuration option ([#4](https://github.com/enpaul/tox-poetry-installer/issues/4))
 - [X] Add per-environment Tox configuration option to fall back to default installation
       backend.
@@ -450,12 +447,13 @@ for usage in production environments.
       Poetry backend. ([#5](https://github.com/enpaul/tox-poetry-installer/issues/5))
 - [X] Add trivial tests to ensure the project metadata is consistent between the pyproject.toml
       and the module constants.
-- [ ] Update to use [poetry-core](https://github.com/python-poetry/poetry-core) and
+- [X] Update to use [poetry-core](https://github.com/python-poetry/poetry-core) and
       improve robustness of the Tox and Poetry module imports
       to avoid potentially breaking API changes in upstream packages. ([#2](https://github.com/enpaul/tox-poetry-installer/issues/2))
 - [ ] Find and implement a way to mitigate the [UNSAFE_DEPENDENCIES issue](https://github.com/python-poetry/poetry/issues/1584) in Poetry.
       ([#6](https://github.com/enpaul/tox-poetry-installer/issues/6))
-- [ ] Fix logging to make proper use of Tox's logging reporter infrastructure
+- [ ] Fix logging to make proper use of Tox's logging reporter infrastructure ([#3](https://github.com/enpaul/tox-poetry-installer/issues/3))
+- [ ] Add configuration option for installing all dev-dependencies to a testenv ([#14](https://github.com/enpaul/tox-poetry-installer/issues/14))
 
 ### Path to Stable
 
@@ -463,6 +461,6 @@ Everything in Beta plus...
 
 - [ ] Add tests for each feature version of Tox between 2.3 and 3.20
 - [ ] Add tests for Python-3.6, 3.7, and 3.8
-- [ ] Add Github Actions based CI
+- [X] Add Github Actions based CI
 - [ ] Add CI for CPython, PyPy, and Conda
 - [ ] Add CI for Linux and Windows

pyproject.toml

@@ -1,11 +1,14 @@
 [tool.poetry]
 name = "tox-poetry-installer"
-version = "0.2.3"
+version = "0.3.1"
 license = "MIT"
 authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 description = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
 repository = "https://github.com/enpaul/tox-poetry-installer/"
-packages = [{include = "tox_poetry_installer.py"}]
+packages = [
+  {include = "tox_poetry_installer.py"},
+  {include = "tests/*.py", format = "sdist"}
+]
 keywords = ["tox", "poetry", "plugin"]
 readme = "README.md"
 classifiers = [
@@ -27,7 +30,8 @@ poetry_installer = "tox_poetry_installer"
 
 [tool.poetry.dependencies]
 python = "^3.6"
-poetry = ">=1.0.0, <1.1.0"
+poetry = "^1.0.0"
+poetry-core = "^1.0.0"
 tox = "^2.3.0 || ^3.0.0"
 
 [tool.poetry.dev-dependencies]

tox.ini

@@ -45,8 +45,8 @@ allowlist_externals =
 commands =
     black {toxinidir}/tests/
     bash -c "reorder-python-imports {toxinidir}/tests/*.py --unclassifiable-application-module tox_poetry_installer"
-    pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tests/
-    mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tests/
+    bash -c "pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tests/*.py"
+    bash -c "mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tests/*.py"
 
 [testenv:security]
 description = Security checks

tox_poetry_installer.py

@@ -1,9 +1,24 @@
 """Tox plugin for installing environments using Poetry
 
-This plugin makes use of the ``tox_testenv_install_deps`` Tox plugin hook to replace the default
+This plugin makes use of the ``tox_testenv_install_deps`` Tox plugin hook to augment the default
 installation functionality to install dependencies from the Poetry lockfile for the project. It
 does this by using ``poetry`` to read in the lockfile, identify necessary dependencies, and then
 use Poetry's ``PipInstaller`` class to install those packages into the Tox environment.
+
+Quick definition of terminology:
+
+* "project package" - the package that Tox is testing, usually the one the current project is
+  is developing; definitionally, this is the package that is built by Tox in the ``.package`` env.
+* "project package dependency" or "project dependency" - a dependency required by the project
+  package for installation; i.e. a package that would be installed when running
+  ``pip install <project package>``.
+* "environment dependency" - a dependency specified for a given testenv in the Tox configuration.
+* "locked dependency" - a package that is present in the Poetry lockfile and will be installed
+  according to the metadata in the lockfile.
+* "unlocked dependency" - a package that is either not present in the Poetry lockfile or is not
+  specified to be installed according to the metadata in the lockfile.
+* "transiety dependency" - a package not explicitly specified for installation, but required by a
+  package that is explicitly specified.
 """
 from pathlib import Path
 from typing import Dict
@@ -13,11 +28,11 @@ from typing import Sequence
 from typing import Set
 from typing import Tuple
 
+from poetry.core.packages import Package as PoetryPackage
 from poetry.factory import Factory as PoetryFactory
-from poetry.factory import Poetry
 from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
 from poetry.io.null_io import NullIO as PoetryNullIO
-from poetry.packages import Package as PoetryPackage
+from poetry.poetry import Poetry
 from poetry.puzzle.provider import Provider as PoetryProvider
 from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
 from tox import hookimpl
@@ -30,7 +45,7 @@ from tox.venv import VirtualEnv as ToxVirtualEnv
 
 __title__ = "tox-poetry-installer"
 __summary__ = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
-__version__ = "0.2.3"
+__version__ = "0.3.1"
 __url__ = "https://github.com/enpaul/tox-poetry-installer/"
 __license__ = "MIT"
 __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
@@ -52,6 +67,10 @@ _REPORTER_PREFIX = f"[{__title__}]:"
 _MAGIC_SUFFIX_MARKER = "@poetry"
 
 
+# Map of package names to the package object
+PackageMap = Dict[str, PoetryPackage]
+
+
 class _SortedEnvDeps(NamedTuple):
     unlocked_deps: List[ToxDepConfig]
     locked_deps: List[ToxDepConfig]
@@ -69,6 +88,10 @@ class LockedDepNotFoundError(ToxPoetryInstallerException):
     """Locked dependency was not found in the lockfile"""
 
 
+class ExtraNotFoundError(ToxPoetryInstallerException):
+    """Project package extra not defined in project's pyproject.toml"""
+
+
 def _sort_env_deps(venv: ToxVirtualEnv) -> _SortedEnvDeps:
     """Sorts the environment dependencies by lock status
 
@@ -111,10 +134,10 @@ def _sort_env_deps(venv: ToxVirtualEnv) -> _SortedEnvDeps:
                 unlocked_deps.append(dep)
 
     reporter.verbosity1(
-        f"{_REPORTER_PREFIX} identified {len(locked_deps)} locked env dependencies for installation from poetry lockfile: {[item.name for item in locked_deps]}"
+        f"{_REPORTER_PREFIX} identified {len(locked_deps)} locked env dependencies: {[item.name for item in locked_deps]}"
     )
     reporter.verbosity1(
-        f"{_REPORTER_PREFIX} identified {len(unlocked_deps)} unlocked env dependencies for installation using default backend: {[item.name for item in unlocked_deps]}"
+        f"{_REPORTER_PREFIX} identified {len(unlocked_deps)} unlocked env dependencies: {[item.name for item in unlocked_deps]}"
     )
 
     return _SortedEnvDeps(locked_deps=locked_deps, unlocked_deps=unlocked_deps)
@@ -129,6 +152,11 @@ def _install_to_venv(
     :param venv: Tox virtual environment to install the packages to
     :param packages: List of packages to install to the virtual environment
     """
+
+    reporter.verbosity1(
+        f"{_REPORTER_PREFIX} Installing {len(packages)} packages to environment at {venv.envconfig.envdir}"
+    )
+
     installer = PoetryPipInstaller(
         env=PoetryVirtualEnv(path=Path(venv.envconfig.envdir)),
         io=PoetryNullIO(),
@@ -140,7 +168,7 @@ def _install_to_venv(
         installer.install(dependency)
 
 
-def _find_transients(poetry: Poetry, dependency_name: str) -> Set[PoetryPackage]:
+def _find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPackage]:
     """Using a poetry object identify all dependencies of a specific dependency
 
     :param poetry: Populated poetry object which can be used to build a populated locked
@@ -152,10 +180,6 @@ def _find_transients(poetry: Poetry, dependency_name: str) -> Set[PoetryPackage]
     .. note:: The package corresponding to the dependency named by ``dependency_name`` is included
               in the list of returned packages.
     """
-    packages: Dict[str, PoetryPackage] = {
-        package.name: package
-        for package in poetry.locker.locked_repository(True).packages
-    }
 
     try:
         top_level = packages[dependency_name]
@@ -163,7 +187,7 @@ def _find_transients(poetry: Poetry, dependency_name: str) -> Set[PoetryPackage]
         def find_deps_of_deps(name: str) -> List[PoetryPackage]:
             if name in PoetryProvider.UNSAFE_PACKAGES:
                 reporter.warning(
-                    f"{_REPORTER_PREFIX} installing '{name}' using Poetry is not supported; skipping"
+                    f"{_REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
                 )
                 return []
             transients = [packages[name]]
@@ -185,20 +209,31 @@ def _find_transients(poetry: Poetry, dependency_name: str) -> Set[PoetryPackage]
         ) from None
 
 
-def _install_env_dependencies(venv: ToxVirtualEnv, poetry: Poetry):
+def _install_env_dependencies(
+    venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
+):
+    """Install the packages for a specified testenv
+
+    Processes the tox environment config, identifies any locked environment dependencies, pulls
+    them from the lockfile, and installs them to the virtual environment.
+
+    :param venv: Tox virtual environment to install the packages to
+    :param poetry: Poetry object the packages were sourced from
+    :param packages: Mapping of package names to the corresponding package object
+    """
     env_deps = _sort_env_deps(venv)
 
     dependencies: List[PoetryPackage] = []
     for dep in env_deps.locked_deps:
         try:
-            dependencies += _find_transients(poetry, dep.name.lower())
+            dependencies += _find_transients(packages, dep.name.lower())
         except ToxPoetryInstallerException as err:
             venv.status = "lockfile installation failed"
             reporter.error(f"{_REPORTER_PREFIX} {err}")
             raise err
 
     reporter.verbosity1(
-        f"{_REPORTER_PREFIX} identified {len(dependencies)} actual dependencies from {len(venv.envconfig.deps)} specified env dependencies"
+        f"{_REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(env_deps.locked_deps)} locked env dependencies"
     )
 
     reporter.verbosity1(
@@ -212,21 +247,55 @@ def _install_env_dependencies(venv: ToxVirtualEnv, poetry: Poetry):
     _install_to_venv(poetry, venv, dependencies)
 
 
-def _install_package_dependencies(venv: ToxVirtualEnv, poetry: Poetry):
+def _install_project_dependencies(
+    venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
+):
+    """Install the dependencies of the project package
+
+    Install all primary dependencies of the project package.
+
+    :param venv: Tox virtual environment to install the packages to
+    :param poetry: Poetry object the packages were sourced from
+    :param packages: Mapping of package names to the corresponding package object
+    """
     reporter.verbosity1(
         f"{_REPORTER_PREFIX} performing installation of project dependencies"
     )
 
-    primary_dependencies = poetry.locker.locked_repository(False).packages
+    base_dependencies: List[PoetryPackage] = [
+        packages[item.name]
+        for item in poetry.package.requires
+        if not item.is_optional()
+    ]
+
+    extra_dependencies: List[PoetryPackage] = []
+    for extra in venv.envconfig.extras:
+        try:
+            extra_dependencies += [
+                packages[item.name] for item in poetry.package.extras[extra]
+            ]
+        except KeyError:
+            raise ExtraNotFoundError(
+                f"Environment '{venv.name}' specifies project extra '{extra}' which was not found in the lockfile"
+            ) from None
+
+    dependencies: List[PoetryPackage] = []
+    for dep in base_dependencies + extra_dependencies:
+        try:
+            dependencies += _find_transients(packages, dep.name.lower())
+        except ToxPoetryInstallerException as err:
+            venv.status = "lockfile installation failed"
+            reporter.error(f"{_REPORTER_PREFIX} {err}")
+            raise err
 
     reporter.verbosity1(
-        f"{_REPORTER_PREFIX} identified {len(primary_dependencies)} dependencies of project '{poetry.package.name}'"
+        f"{_REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies"
     )
 
     reporter.verbosity0(
-        f"{_REPORTER_PREFIX} ({venv.name}) installing {len(primary_dependencies)} project dependencies from lockfile"
+        f"{_REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile"
     )
-    _install_to_venv(poetry, venv, primary_dependencies)
+    _install_to_venv(poetry, venv, dependencies)
 
 
 @hookimpl
@@ -283,19 +352,26 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction):
         f"{_REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}"
     )
 
+    package_map: PackageMap = {
+        package.name: package
+        for package in poetry.locker.locked_repository(True).packages
+    }
+
     # Handle the installation of any locked env dependencies from the lockfile
-    _install_env_dependencies(venv, poetry)
+    _install_env_dependencies(venv, poetry, package_map)
 
     # Handle the installation of the package dependencies from the lockfile if the package is
     # being installed to this venv; otherwise skip installing the package dependencies
-    if not venv.envconfig.skip_install and not venv.envconfig.config.skipsdist:
-        _install_package_dependencies(venv, poetry)
-    else:
     if venv.envconfig.skip_install:
         reporter.verbosity1(
             f"{_REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package"
         )
-        elif venv.envconfig.config.skipsdist:
+        return
+
+    if venv.envconfig.config.skipsdist:
         reporter.verbosity1(
             f"{_REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package"
         )
+        return
+
+    _install_project_dependencies(venv, poetry, package_map)