Merge pull request #77 from enpaul/enp/0.10.0

Prepare for 0.10 release

.github/scripts/setup-env.sh

@@ -8,7 +8,7 @@
 set -e;
 
 CI_CACHE=$HOME/.cache;
-POETRY_VERSION=1.1.12;
+POETRY_VERSION=1.2.0;
 
 mkdir --parents "$CI_CACHE";
 

.github/workflows/ci.yaml

@@ -7,12 +7,11 @@ on:
     branches: ["devel"]
 jobs:
   Test:
+    name: Python ${{ matrix.python.version }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python:
-          - version: "3.6"
-            toxenv: py36
           - version: "3.7"
             toxenv: py37
           - version: "3.8"
@@ -21,15 +20,18 @@ jobs:
             toxenv: py39
           - version: "3.10"
             toxenv: py310
+      fail-fast: true
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+
       - name: Install Python ${{ matrix.python.version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python.version }}
+
       - name: Configure Job Cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/pip
@@ -39,38 +41,49 @@ jobs:
           # will be invalidated, and thus all packages will be redownloaded, if the
           # lockfile is updated
           key: ${{ runner.os }}-${{ matrix.python.toxenv }}-${{ hashFiles('**/poetry.lock') }}
+
       - name: Configure Path
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
       - name: Configure Environment
         run: .github/scripts/setup-env.sh
+
       - name: Run Toxenv ${{ matrix.python.toxenv }}
         run: poetry run tox -e ${{ matrix.python.toxenv }}
+
   Check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Install Python 3.8
-        uses: actions/setup-python@v1
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
+
       - name: Configure Job Cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/pip
             ~/.cache/pypoetry/cache
             ~/.poetry
-          # Hardcoded 'py38' slug here lets this cache piggyback on the 'py38' cache
+          # Hardcoded 'py310' slug here lets this cache piggyback on the 'py310' cache
           # that is generated for the tests above
-          key: ${{ runner.os }}-py38-${{ hashFiles('**/poetry.lock') }}
+          key: ${{ runner.os }}-py310-${{ hashFiles('**/poetry.lock') }}
+
       - name: Configure Path
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
       - name: Configure Environment
         run: .github/scripts/setup-env.sh
+
       - name: Run Static Analysis Checks
         run: poetry run tox -e static
+
       - name: Run Static Analysis Checks (Tests)
         run: poetry run tox -e static-tests
+
       - name: Run Security Checks
         run: poetry run tox -e security

CHANGELOG.md

@@ -2,6 +2,38 @@
 
 See also: [Github Release Page](https://github.com/enpaul/tox-poetry-installer/releases).
 
+## Version 0.10.0
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.10.0),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.10.0/)
+
+- Add `poetry_dep_groups` option to support installing groups of Poetry dependencies.
+  Contributed by [Oshmoun](https://github.com/oshmoun) (#76)
+- Deprecate `install_dev_deps` option
+
+## Version 0.9.0
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.9.0),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.9.0/)
+
+- Add support for Poetry-1.2.x. Contributed by [Justin Wood](https://github.com/Callek)
+  (#73)
+- Update Black formatting to stable release version
+- Remove support for Python-3.6
+- Remove support for Poetry-1.1.x
+- Fix installing dependencies multiple times when transient dependencies are duplicated in
+  the dependency tree
+
+## Version 0.8.5
+
+View this release on:
+[Github](https://github.com/enpaul/tox-poetry-installer/releases/tag/0.8.5),
+[PyPI](https://pypi.org/project/tox-poetry-installer/0.8.5/)
+
+- Fix Poetry version specification supporting the incompatible Poetry-1.2.0 release
+
 ## Version 0.8.4
 
 View this release on:
@@ -27,7 +59,7 @@ View this release on:
 
 - Improve debug-level logging for package installation, and time how long installing each
   package takes. Contributed by [Rebecca
-  Turner](https://github.com/9999years).
+  Turner](https://github.com/9999years) (#63).
 - Fix crash caused by the package-under-test depending on Poetry's unsafe dependencies ([#65](https://github.com/enpaul/tox-poetry-installer/issues/65))
 
 ## Version 0.8.1

Makefile

@@ -33,7 +33,7 @@ test: ## Run the project testsuite(s)
 	poetry run tox --recreate
 
 dev: ## Create the local dev environment
-	poetry install -E poetry
+	poetry install -E poetry --sync
 	poetry run pre-commit install
 
 publish: test wheel source ## Build and upload to pypi (requires $PYPI_API_KEY be set)

README.md

@@ -34,16 +34,16 @@ See the
 [Changelog](https://github.com/enpaul/tox-poetry-installer/blob/devel/CHANGELOG.md) for
 release history.
 
-*See also: [official Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html), [Poetry-Dev-Dependencies Tox plugin](https://github.com/sinoroc/tox-poetry-dev-dependencies), [Poetry Tox plugin](https://github.com/tkukushkin/tox-poetry)*
+*See also: [official Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html), [Poetry Tox plugin](https://github.com/tkukushkin/tox-poetry)*
 
 ## Feature Overview
 
 - Manage package versions in exactly one place and with exactly one tool: Poetry.
 - Ensure CI/CD and other automation tools are using the same package versions that you are
   in your local development environment.
-- Add only the packages you need to a Tox test environment, instead of everything in your
-  lockfile.
-- Directly integrate with Poetry, re-using your existing package indexes and credentials
+- Add only the packages or custom groups you need to a Tox test environment, instead of
+  everything in your lockfile.
+- Directly integrate with Poetry, re-using your existing package indexes and credentials,
   with no additional configuration.
 - Wherever possible, built-in Tox config options are always respected and their behavior
   kept consistent.
@@ -151,19 +151,23 @@ commands = ...
 > the child environment with a different value.
 
 Alternatively, we can skip specifying all of our dependencies for a test environment in
-the Tox config and just install all of our Poetry dev-dependencies automatically:
+the Tox config and install Poetry dependency groups directly:
 
 ```ini
 [testenv]
 description = Some very cool tests
 require_locked_deps = true
-install_dev_deps = true
+poetry_dep_groups =
+    dev
 commands = ...
 ```
 
-> **Note:** Setting `install_dev_deps = true` on an environment that also installs the
+> **Note:** Setting `poetry_dep_groups = [dev]` on an environment that also installs the
 > project package is functionally equivalent to running `poetry install`.
 
+> **Note:** The `install_dev_deps` configuration option is deprecated. See \[Configuration
+> Options\](#configuration-options\] for more information.
+
 Finally, we can also install an unlocked dependency (a dependency which doesn't take its
 version from the Poetry lockfile) into the test environment alongside the locked ones. We
 need to remove the `require_locked_deps = true` option, otherwise the environment will
@@ -198,9 +202,13 @@ configuration section.
 | :--------------------- | :-----: | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `locked_deps`          |  List   |  `[]`   | Names of packages to install to the test environment from the Poetry lockfile. Transient dependencies (packages required by these dependencies) are automatically included.                                                                                                                                                                                          |
 | `require_locked_deps`  | Boolean |  False  | Whether the plugin should block attempts to install unlocked dependencies to the test environment. If enabled, then the [`tox_testenv_install_deps`](https://tox.readthedocs.io/en/latest/plugins.html#tox.hookspecs.tox_testenv_install_deps) plugin hook will be intercepted and an error will be raised if the test environment has the `deps` option configured. |
-| `install_dev_deps`     | Boolean |  False  | Whether all of the Poetry dev-dependencies should be installed to the test environment.                                                                                                                                                                                                                                                                              |
 | `install_project_deps` | Boolean |  True   | Whether all of the Poetry primary dependencies for the project package should be installed to the test environment.                                                                                                                                                                                                                                                  |
 | `require_poetry`       | Boolean |  False  | Whether Tox should be forced to fail if the plugin cannot import Poetry locally. If `False` then the plugin will be skipped for the test environment if Poetry cannot be imported. If `True` then the plugin will force the environment to error and the Tox run to fail.                                                                                            |
+| `poetry_dep_groups`    |  List   |  `[]`   | Names of Poetry dependency groups specified in `pyproject.toml` to install to the test environment.                                                                                                                                                                                                                                                                  |
+
+> **Note:** The `install_dev_deps` configuration option is deprecated and will be removed in
+> version 1.0.0. Please set `poetry_dep_groups = [dev]` in `tox.ini` for environments that
+> install the development dependencies.
 
 ### Runtime Options
 
@@ -230,7 +238,7 @@ error will be set to one of the "Status" values below to indicate what the error
 | `LockedDepVersionConflictError` | Indicates that an item in the `locked_deps` config option includes a [PEP-508 version specifier](https://www.python.org/dev/peps/pep-0508/#grammar) (ex: `pytest >=6.0, <6.1`).                                                 |
 | `LockedDepNotFoundError`        | Indicates that an item specified in the `locked_deps` config option does not match the name of a package in the Poetry lockfile.                                                                                                |
 | `LockedDepsRequiredError`       | Indicates that a test environment with the `require_locked_deps` config option set to `true` also specified unlocked dependencies using the [`deps`](https://tox.readthedocs.io/en/latest/config.html#conf-deps) config option. |
-| `PoetryNotInstalledError`       | Indicates that the `poetry` module could not be imported under the current runtime environment, and the `--require-poetry` flag was provided.                                                                                   |
+| `PoetryNotInstalledError`       | Indicates that the `poetry` module could not be imported under the current runtime environment, and `require_poetry = true` was specified.                                                                                      |
 | `RequiresUnsafeDepError`        | Indicates that the package-under-test depends on a package that Poetry has classified as unsafe and cannot be installed.                                                                                                        |
 
 > **Note:** One or more of these errors can be caused by the `pyproject.toml` being out of
@@ -353,7 +361,7 @@ are tracked on [Github](https://github.com/enpaul/tox-poetry-installer/releases)
   and [open a pull request](https://github.com/enpaul/tox-poetry-installer/compare).
 
 Developing this project requires [Python 3.7+](https://www.python.org/downloads/) and
-[Poetry 1.0](https://python-poetry.org/docs/#installation) or later. GNU Make can
+[Poetry 1.2](https://python-poetry.org/docs/#installation) or later. GNU Make can
 optionally be used to quickly setup a local development environment, but this is not
 required.
 
@@ -368,18 +376,11 @@ git clone git@github.com:enpaul/tox-poetry-installer.git
 
 cd tox-poetry-installer/
 
-# Create and configure the local development environment...
-# ...with make:
+# Create and configure the local development environment
 make dev
-# ...manually:
-poetry install -E poetry --remove-untracked
-poetry run pre-commit install
 
-# Run tests and CI locally...
-# ...with make:
+# Run tests and CI locally
 make test
-# ...manually:
-poetry run tox --recreate
 
 # See additional make targets
 make help

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tox-poetry-installer"
-version = "0.8.4"
+version = "0.10.0"
 license = "MIT"
 authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 description = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile"
@@ -23,7 +23,6 @@ classifiers = [
   "Natural Language :: English",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.6",
   "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
@@ -35,25 +34,26 @@ classifiers = [
 poetry_installer = "tox_poetry_installer"
 
 [tool.poetry.extras]
-poetry = ["poetry"]
+poetry = ["poetry", "cleo"]
 
 [tool.poetry.dependencies]
-python = "^3.6.1"
-poetry = {version = "^1.0.0", optional = true}
-poetry-core = "^1.0.0"
+python = "^3.7"
+cleo = {version = "^1.0.0a5", optional = true, allow-prereleases = true}
+poetry = {version = "^1.2.0", optional = true}
+poetry-core = "^1.1.0"
 tox = "^3.8.0"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 bandit = "^1.6.2"
-black = {version = "^21.12b0", allow-prereleases = true, python = "^3.7"}
+black = "^22.3.0"
 blacken-docs = "^1.8.0"
-ipython = { version = "^7.18.1", python = "^3.7" }
+ipython = "^7.18.1"
 mdformat = "^0.6"
 mdformat-gfm = "^0.2"
 mypy = "^0.930"
 pre-commit = "^2.7.1"
 pre-commit-hooks = "^3.3.0"
-pylint = "^2.4.4"
+pylint = "^2.13.0"
 pytest = "^6.0.2"
 pytest-cov = "^2.10.1"
 reorder-python-imports = "^2.3.5"
@@ -63,5 +63,5 @@ tox = "^3.20.0"
 types-toml = "^0.10.1"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=1.1.0"]
 build-backend = "poetry.core.masonry.api"

tests/fixtures.py

@@ -7,7 +7,7 @@ import poetry.installation.pip_installer
 import poetry.utils.env
 import pytest
 import tox
-from poetry.core.packages import Package as PoetryPackage
+from poetry.core.packages.package import Package as PoetryPackage
 
 from tox_poetry_installer import utilities
 

tests/test_installer.py

@@ -14,7 +14,7 @@ def test_deduplication(mock_venv, mock_poetry_factory):
     """Test that the installer does not install duplicate dependencies"""
     poetry = Factory().create_poetry(None)
     packages: utilities.PackageMap = {
-        item.name: item for item in poetry.locker.locked_repository(False).packages
+        item.name: item for item in poetry.locker.locked_repository().packages
     }
 
     venv = tox.venv.VirtualEnv()
@@ -29,7 +29,7 @@ def test_parallelization(mock_venv, mock_poetry_factory):
     """Test that behavior is consistent between parallel and non-parallel usage"""
     poetry = Factory().create_poetry(None)
     packages: utilities.PackageMap = {
-        item.name: item for item in poetry.locker.locked_repository(False).packages
+        item.name: item for item in poetry.locker.locked_repository().packages
     }
 
     to_install = [

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36, py37, py38, py39, py310, static, static-tests, security
+envlist = py37, py38, py39, py310, static, static-tests, security
 isolated_build = true
 skip_missing_interpreters = true
 
@@ -21,7 +21,7 @@ commands =
 
 [testenv:static]
 description = Static formatting and quality enforcement
-basepython = python3.8
+basepython = python3.10
 platform = linux
 ignore_errors = true
 locked_deps =
@@ -46,7 +46,7 @@ commands =
 
 [testenv:static-tests]
 description = Static formatting and quality enforcement for the tests
-basepython = python3.8
+basepython = python3.10
 platform = linux
 ignore_errors = true
 locked_deps =
@@ -63,7 +63,7 @@ commands =
 
 [testenv:security]
 description = Security checks
-basepython = python3.8
+basepython = python3.10
 platform = linux
 ignore_errors = true
 skip_install = true

tox_poetry_installer/__about__.py

@@ -1,7 +1,7 @@
 # pylint: disable=missing-docstring
 __title__ = "tox-poetry-installer"
 __summary__ = "A plugin for Tox that lets you install test environment dependencies from the Poetry lockfile"
-__version__ = "0.8.4"
+__version__ = "0.10.0"
 __url__ = "https://github.com/enpaul/tox-poetry-installer/"
 __license__ = "MIT"
 __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]

tox_poetry_installer/_poetry.py

@@ -28,9 +28,9 @@ from tox_poetry_installer import exceptions
 
 
 try:
+    from cleo.io.null_io import NullIO
     from poetry.factory import Factory
     from poetry.installation.pip_installer import PipInstaller
-    from poetry.io.null_io import NullIO
     from poetry.poetry import Poetry
     from poetry.utils.env import VirtualEnv
 except ImportError:

tox_poetry_installer/constants.py

@@ -20,7 +20,8 @@ PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<")
 REPORTER_PREFIX: str = f"{__about__.__title__}:"
 
 # Internal list of packages that poetry has deemed unsafe and are excluded from the lockfile
-UNSAFE_PACKAGES: Set[str] = {"distribute", "pip", "setuptools", "wheel"}
+# TODO: This functionality is no longer needed, should remove in a future update.
+UNSAFE_PACKAGES: Set[str] = set()
 
 # Number of threads to use for installing dependencies by default
 DEFAULT_INSTALL_THREADS: int = 10

tox_poetry_installer/hooks.py

@@ -4,6 +4,7 @@ All implementations of tox hooks are defined here, as well as any single-use hel
 specifically related to implementing the hooks (to keep the size/readability of the hook functions
 themselves manageable).
 """
+from itertools import chain
 from typing import Optional
 
 import tox
@@ -114,7 +115,14 @@ def tox_addoption(parser: ToxParser):
         name="install_dev_deps",
         type="bool",
         default=False,
-        help="Automatically install all Poetry development dependencies to the environment",
+        help="(deprecated) Automatically install all Poetry development dependencies to the environment",
+    )
+
+    parser.add_testenv_attribute(
+        name="poetry_dep_groups",
+        type="line-list",
+        default=[],
+        help="List of Poetry dependency groups to install to the environment",
     )
 
     parser.add_testenv_attribute(
@@ -196,6 +204,20 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
             dev_deps = []
             logger.info("Env does not install development dependencies, skipping")
 
+        group_deps = utilities.dedupe_packages(
+            list(
+                chain(
+                    *[
+                        utilities.find_group_deps(group, packages, virtualenv, poetry)
+                        for group in venv.envconfig.poetry_dep_groups
+                    ]
+                )
+            )
+        )
+        logger.info(
+            f"Identified {len(group_deps)} group dependencies to install to env"
+        )
+
         env_deps = utilities.find_additional_deps(
             packages, virtualenv, poetry, venv.envconfig.locked_deps
         )
@@ -231,7 +253,9 @@ def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional
         logger.error(f"Internal plugin error: {err}")
         raise err
 
-    dependencies = dev_deps + env_deps + project_deps
+    dependencies = utilities.dedupe_packages(
+        dev_deps + group_deps + env_deps + project_deps
+    )
     if (
         venv.envconfig.config.option.parallel_install_threads
         != constants.DEFAULT_INSTALL_THREADS

tox_poetry_installer/installer.py

@@ -6,10 +6,10 @@ import concurrent.futures
 import contextlib
 import typing
 from datetime import datetime
-from typing import Sequence
+from typing import Collection
 from typing import Set
 
-from poetry.core.packages import Package as PoetryPackage
+from poetry.core.packages.package import Package as PoetryPackage
 from tox.venv import VirtualEnv as ToxVirtualEnv
 
 from tox_poetry_installer import logger
@@ -22,7 +22,7 @@ if typing.TYPE_CHECKING:
 def install(
     poetry: "_poetry.Poetry",
     venv: ToxVirtualEnv,
-    packages: Sequence[PoetryPackage],
+    packages: Collection[PoetryPackage],
     parallels: int = 0,
 ):
     """Install a bunch of packages to a virtualenv

tox_poetry_installer/utilities.py

@@ -10,8 +10,8 @@ from typing import List
 from typing import Sequence
 from typing import Set
 
-from poetry.core.packages import Dependency as PoetryDependency
-from poetry.core.packages import Package as PoetryPackage
+from poetry.core.packages.dependency import Dependency as PoetryDependency
+from poetry.core.packages.package import Package as PoetryPackage
 from tox.action import Action as ToxAction
 from tox.venv import VirtualEnv as ToxVirtualEnv
 
@@ -59,6 +59,13 @@ def check_preconditions(venv: ToxVirtualEnv, action: ToxAction) -> "_poetry.Poet
             "be removed in version 1.0.0. Please use the '--parallel-install-threads' option."
         )
 
+    if venv.envconfig.install_dev_deps:
+        logger.warning(
+            "DEPRECATION: The 'install_dev_deps' option is deprecated and will be removed in "
+            "version 1.0.0. Please update test environments that install development dependencies "
+            "to set the 'poetry_dev_groups = [dev]' option in tox.ini"
+        )
+
     from tox_poetry_installer import _poetry
 
     try:
@@ -92,7 +99,7 @@ def build_package_map(poetry: "_poetry.Poetry") -> PackageMap:
     :returns: Mapping of package names to Poetry package objects
     """
     packages = collections.defaultdict(list)
-    for package in poetry.locker.locked_repository(True).packages:
+    for package in poetry.locker.locked_repository().packages:
         packages[package.name].append(package)
 
     return packages
@@ -218,7 +225,7 @@ def find_project_deps(
             dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
         )
 
-    return dependencies
+    return dedupe_packages(dependencies)
 
 
 def find_additional_deps(
@@ -243,7 +250,34 @@ def find_additional_deps(
             dep_name.lower(), packages, venv, allow_missing=[poetry.package.name]
         )
 
-    return dependencies
+    return dedupe_packages(dependencies)
+
+
+def find_group_deps(
+    group: str,
+    packages: PackageMap,
+    venv: "_poetry.VirtualEnv",
+    poetry: "_poetry.Poetry",
+) -> List[PoetryPackage]:
+    """Find the dependencies belonging to a dependency group
+
+    Recursively identify the Poetry dev dependencies
+
+    :param group: Name of the dependency group from the project's ``pyproject.toml``
+    :param packages: Mapping of all locked package names to their corresponding package object
+    :param venv: Poetry virtual environment to use for package compatibility checks
+    :param poetry: Poetry object for the current project
+    """
+    return find_additional_deps(
+        packages,
+        venv,
+        poetry,
+        poetry.pyproject.data["tool"]["poetry"]
+        .get("group", {})
+        .get(group, {})
+        .get("dependencies", {})
+        .keys(),
+    )
 
 
 def find_dev_deps(
@@ -257,9 +291,26 @@ def find_dev_deps(
     :param venv: Poetry virtual environment to use for package compatibility checks
     :param poetry: Poetry object for the current project
     """
-    return find_additional_deps(
+    dev_group_deps = find_group_deps("dev", packages, venv, poetry)
+
+    # Legacy pyproject.toml poetry format:
+    legacy_dev_group_deps = find_additional_deps(
         packages,
         venv,
         poetry,
         poetry.pyproject.data["tool"]["poetry"].get("dev-dependencies", {}).keys(),
     )
+
+    # Poetry 1.2 unions these two toml sections.
+    return dedupe_packages(dev_group_deps + legacy_dev_group_deps)
+
+
+def dedupe_packages(packages: Sequence[PoetryPackage]) -> List[PoetryPackage]:
+    """Deduplicates a sequence of PoetryPackages while preserving ordering
+
+    Adapted from StackOverflow: https://stackoverflow.com/a/480227
+    """
+    seen: Set[PoetryPackage] = set()
+    # Make this faster, avoid method lookup below
+    seen_add = seen.add
+    return [p for p in packages if not (p in seen or seen_add(p))]