84 Commits
0.1.2 ... 0.5.0

Author SHA1 Message Date
b9b0eba90f Merge pull request #26 from enpaul/enp/fixups
Minor fixups ahead of 0.5 release
2020-11-12 19:47:09 -05:00
df8312f5ee Bump feature version 2020-11-12 19:42:43 -05:00
1f102b16cb Add blocking functionality when using require_locked_deps
When require_locked_deps is true further processing will be blocked because
a non-null value is returned by this function
2020-11-12 19:42:43 -05:00
a7cde7a9ab Fix missing exception docs 2020-11-12 19:42:43 -05:00
b0bee11272 Merge pull request #25 from enpaul/enp/docs
Update documentation ahead of 0.5 release
2020-11-12 19:42:27 -05:00
accb4c3278 Update drawbacks section with new info 2020-11-12 19:31:31 -05:00
ea183553c4 Rewrite usage examples section into reference section
Fixes #20
2020-11-12 19:31:30 -05:00
5c5536581b Update quickstart docs with new config option usage with locked_deps 2020-11-12 18:45:02 -05:00
b6ef671e67 Merge pull request #23 from enpaul/enp/faster
Update config handling to support tox's native env change detection
2020-11-12 18:19:21 -05:00
99c10482fc Update tox config to use new plugin config option 2020-11-12 00:54:27 -05:00
b32a212e82 Restructure config options to support tox's native change detection
Remove custom handling of deps option
Add locked_deps option
Stop modifying the envconfig.deps option at runtime
2020-11-12 00:54:27 -05:00
b6415888d9 Merge pull request #24 from enpaul/enp/fix-ci
Fix automated CI
2020-11-12 00:52:18 -05:00
eba268a127 Update CI to include installation of this plugin
Not sure how I overlooked this
2020-11-12 00:44:12 -05:00
d8d483e849 Update to use poetry-core for PEP517 builds 2020-11-12 00:41:54 -05:00
c14313b7b0 Fix overlooked roadmap step that didn't get checked off 2020-11-12 00:32:22 -05:00
d1f161e0fa Merge pull request #22 from enpaul/enp/recursive
Fix recursive dependencies causing recursion error
2020-11-12 00:26:22 -05:00
2961b55c9a Fix recursive dependencies causing recursion error
Rewrite the find_dep_of_deps function to handle recursive dependencies
2020-11-11 23:07:14 -05:00
d4fb7046d8 Merge pull request #21 from enpaul/enp/modsplit
Split single-file module out into directory module
2020-11-11 22:49:30 -05:00
8c4e596316 Update tox automation to work with new module structure 2020-11-11 22:37:09 -05:00
50f6e3d151 Update dependencies to get cryptography security fix 2020-11-11 22:37:09 -05:00
106d1bf6cf Update metadata tests to load metadata from new location 2020-11-11 22:37:09 -05:00
b57b78d4e2 Add typing stub file 2020-11-11 22:37:08 -05:00
33e81f742a Split single module file out into directory module 2020-11-11 22:37:08 -05:00
3a262d718c Merge pull request #19 from enpaul/enp/install-dev-deps
Add config option for installing dev dependencies to testenv
2020-10-29 17:28:39 -04:00
5979ec7a8a Add documentation for new config option 2020-10-28 16:26:31 -04:00
c7bb3d35ea Bump feature version 2020-10-24 12:10:07 -04:00
961e6f6acd Add support for installing all dev dependencies to a testenv 2020-10-24 12:09:42 -04:00
ed039de674 Merge pull request #16 from enpaul/enp/fix
Fix support for extras installation
2020-10-24 11:55:35 -04:00
db0cf6ce0c Bump patch version 2020-10-24 11:53:38 -04:00
e8d3f4fcac Fix support for non-single extra dependency installation 2020-10-24 11:53:38 -04:00
653622fd35 Merge pull request #15 from enpaul/enp/housekeeping
Some housekeeping changes
2020-10-23 10:02:45 -04:00
de4c3515ec Refactor to maintain consistent terminology
With a dozen different "dependency" types, consistent naming is critical
2020-10-23 00:59:43 -04:00
01bfdd74bd Add missing documentation for internal functions/constants 2020-10-23 00:43:11 -04:00
3e98ec81eb Update beta roadmap features list 2020-10-22 21:16:32 -04:00
b6534f86d0 Merge pull request #11 from enpaul/enp/extras
Add support for the extras option
2020-10-22 21:10:18 -04:00
74e3ed01c0 Fix handling of nonexistent extras with custom app exception 2020-10-22 21:07:56 -04:00
f944843278 Bump feature version 2020-10-22 21:07:55 -04:00
7c761073b3 Add poetry-core as explicit dependency to support poetry-1.0 2020-10-22 20:47:46 -04:00
6075ea6a3e Update readme with development progress 2020-10-22 20:47:44 -04:00
fdee2d9a8d Add support for extras tox configuration option
Install only non-optional project dependencies by default
Install dependencies for extras when specified

Fixes #4
2020-10-22 20:47:19 -04:00
699fb347da Add links to other poetry tox plugins 2020-10-22 00:06:37 -04:00
a3bfd2687a Remove obsoleted documentation from readme
Clarify developer poetry requirement
Work on #12
2020-10-22 00:03:19 -04:00
0bf3b16091 Merge pull request #10 from enpaul/enp/core
Support poetry core
2020-10-11 20:15:46 -04:00
979fa58618 Bump patch version 2020-10-11 20:11:42 -04:00
462cc166a9 Include tests in sdist 2020-10-11 20:11:42 -04:00
30160985c1 Update poetry imports to use core module 2020-10-11 20:11:42 -04:00
5d87ffac72 Update poetry requirement to support 1.1+ 2020-10-11 20:11:42 -04:00
6b92189e50 !fixup 2020-10-11 20:10:30 -04:00
0fbc77c2c4 Add CI status badge 2020-10-11 20:08:43 -04:00
7867a7c98b Merge pull request #9 from enpaul/enp/ci
Add CI for PRs and CD for default branch
2020-10-11 20:03:12 -04:00
7a34c47168 Wrap test checks in bash to avoid globbing errors 2020-10-11 19:41:16 -04:00
4a1dc52755 Fix python version in pre-commit config 2020-10-11 19:41:16 -04:00
1e2156ecdb Add CI for PRs and CD for default branch 2020-10-11 19:41:16 -04:00
707a73c922 Bump patch version 2020-09-29 19:01:59 -04:00
640dbfe102 Fix implicit case-sensitive locked dependency name handling
Fixes #7
2020-09-29 19:01:52 -04:00
05c5a26cc4 Update non-poetry error handling to be more permissive
Fixes #1
2020-09-29 19:01:43 -04:00
ee6f939b6a Fix links to roadmap issues 2020-09-29 00:58:04 -04:00
82678e81e8 Add issue links for relevant roadmap items 2020-09-29 00:56:22 -04:00
5411025612 Proofreading and editing fixes
Check spelling more thoroughly
Improve clarity in a few places
Fix grammar mistakes
Fix docs that didn't get updated for the 0.2 overhaul
2020-09-29 00:51:08 -04:00
2e1d5fc922 Bump patch version 2020-09-28 23:26:19 -04:00
b7961bec58 Add support for running in a non-poetry project
Handle the error poetry raises when not in a poetry managed project
  and use that as a sign to skip further usage of the plugin in that
  environment
2020-09-28 23:24:42 -04:00
e28159060d Add badge links
Add black code format badge
2020-09-27 20:59:41 -04:00
edcef918b3 Rename 'usage' section to 'usage examples' 2020-09-27 19:41:29 -04:00
beba9416be Add note about root env setting inheritance breaking child envs 2020-09-27 19:38:42 -04:00
18a74fab63 Bump patch version 2020-09-27 15:57:00 -04:00
516515b347 Fix duplicate installation of env dependencies
Fix always logging post-sorted unlocked env dependencies
2020-09-27 15:56:29 -04:00
c9f1f41163 Fix installing package deps when skipdist is true 2020-09-27 15:56:29 -04:00
78efd82c82 Add quickstart section 2020-09-27 14:51:12 -04:00
5476f4ab11 Update example errors with new names, messages, and formatting 2020-09-27 14:45:11 -04:00
a4d1c1e4df Add missing drawback about poetry unsafe dependencies
Add item to beta specification to fix/mitigate this somehow
2020-09-27 14:45:11 -04:00
fb1ac3b0de Misc documentation updates
Fix inconsistent env naming in example ini snippets
Fix PS1 in demo commands to make it clearer what they are
Rename 'getting started' section to 'usage'
Add note about main branch and tag usage
2020-09-27 14:44:50 -04:00
c481b7b0bb Update error handling to improve UX
Add discrete exception for version conflict vs not in lockfile errors
Update to set tox venv to failed when conflict happens
Update tox reporting for errors to use proper level
2020-09-27 14:17:15 -04:00
f20e434f2c Overhaul usage documentation
Add better installation documentation
Add configuration examples and usage walk through using new design system
Update roadmap with feature changes for 0.2
Clarify drawbacks section with more useful context
2020-09-27 14:02:39 -04:00
10211bc674 Bump feature version 2020-09-26 10:48:10 -04:00
50c008d054 Require locked dependencies for all envs
Gotta dogfood sometime
2020-09-26 10:47:46 -04:00
476f27943e Fix bug with dependency name modification
Standardize logging messages
2020-09-26 10:47:45 -04:00
8bb9255fc1 Implement new config interface system to expose more options
Default behavior is now to only install project package deps from lockfile
Specific env deps can be locked using @poetry suffix
Entire env can now be forced to use locked deps with require_locked_deps option
2020-09-26 10:46:38 -04:00
66f2c3c768 Bump patch version 2020-09-25 01:04:12 -04:00
fd2637113f Remove excessive bandit output from security checks 2020-09-25 01:03:31 -04:00
b10e796ca1 Standardize log message usage of 'dev-package' and 'env' terminology 2020-09-25 01:02:30 -04:00
5dfbca4ff6 Update docs to indicate dev package installation support 2020-09-25 00:56:21 -04:00
db09acd8fe Fix install of dev package dependencies from lockfile
Override default pip behavior by preemptively installing dev package dependencies
Keep support for tox default skip_install config flag
2020-09-25 00:54:45 -04:00
b339e3d6d9 Update poetry requirement to mitigate coming breaking API changes
Poetry 1.1 is due any day and when it does much of the functionality this
  module uses will be moved to poetry-core. Until this module is updated
  to use poetry-core 1.1 will be a breaking change
2020-09-25 00:38:20 -04:00
9db6838d94 Update logging calls to use the tox reporter 2020-09-24 23:56:36 -04:00
16 changed files with 1185 additions and 836 deletions

41
.github/workflows/ci.yaml vendored Normal file
View File

@@ -0,0 +1,41 @@
---
name: CI
on:
pull_request:
types: ["opened", "synchronize"]
push:
branches: ["devel"]
jobs:
Test:
runs-on: ubuntu-latest
strategy:
matrix:
python:
- version: 3.6
toxenv: py36
- version: 3.7
toxenv: py37
- version: 3.8
toxenv: py38
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python.version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python.version }}
- name: Install project
run: pip install .
- name: Run tests via ${{ matrix.python.toxenv }}
run: tox -e ${{ matrix.python.toxenv }}
Check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install project
run: pip install .
- name: Run meta checks
run: tox -e static -e static-tests -e security

View File

@@ -4,14 +4,14 @@ repos:
rev: 20.8b1
hooks:
- id: black
language_version: python3.7
language_version: python3
- repo: https://github.com/asottile/blacken-docs
rev: v0.5.0
hooks:
- id: blacken-docs
additional_dependencies: [black==20.8b1]
language_version: python3.7
language_version: python3
- repo: https://github.com/asottile/reorder_python_imports
rev: v2.3.5

332
README.md
View File

@@ -1,19 +1,26 @@
# tox-poetry-installer
A plugin for [Tox](https://tox.readthedocs.io/en/latest/) that allows test environment
dependencies to be installed using [Poetry](https://python-poetry.org/) using its lockfile.
dependencies to be installed using [Poetry](https://python-poetry.org/) from its lockfile.
⚠️ **This project is alpha software and should not be used in a production capacity** ⚠️
⚠️ **This project is alpha software and should not be used in production environments** ⚠️
![image](https://img.shields.io/pypi/l/tox-poetry-installer)
![image](https://img.shields.io/pypi/v/tox-poetry-installer)
![image](https://img.shields.io/pypi/pyversions/tox-poetry-installer)
[![ci-status](https://github.com/enpaul/tox-poetry-installer/workflows/CI/badge.svg?event=push)](https://github.com/enpaul/tox-poetry-installer/actions)
[![license](https://img.shields.io/pypi/l/tox-poetry-installer)](https://opensource.org/licenses/MIT)
[![pypi-version](https://img.shields.io/pypi/v/tox-poetry-installer)](https://pypi.org/project/tox-poetry-installer/)
[![python-versions](https://img.shields.io/pypi/pyversions/tox-poetry-installer)](https://www.python.org)
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
**Documentation**
* [Installation and Usage](#installation-and-usage)
* [Limitations](#limitations)
* [Why would I use this?](#what-problems-does-this-solve) (What problems does this solve?)
* [Installation](#installation)
* [Quick Start](#quick-start)
* [Reference and Usage](#reference-and-usage)
* [Config Option Reference](#config-option-reference)
* [Error Reference](#error-reference)
* [Example Config](#example-config)
* [Known Drawbacks and Problems](#known-drawbacks-and-problems)
* [Why would I use this?](#why-would-i-use-this) (What problems does this solve?)
* [Developing](#developing)
* [Contributing](#contributing)
* [Roadmap](#roadmap)
@@ -23,85 +30,207 @@ dependencies to be installed using [Poetry](https://python-poetry.org/) using it
Related resources:
* [Poetry Python Project Manager](https://python-poetry.org/)
* [Tox Automation Project](https://tox.readthedocs.io/en/latest/)
* [Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html)
* [Poetry Dev-Dependencies Tox Plugin](https://github.com/sinoroc/tox-poetry-dev-dependencies)
* [Poetry Tox Plugin](https://github.com/tkukushkin/tox-poetry)
* [Other Tox plugins](https://tox.readthedocs.io/en/latest/plugins.html)
## Installation and Usage
## Installation
1. Install the plugin from PyPI:
Add the plugin as a development dependency of a Poetry project:
```
poetry add tox-poetry-installer --dev
~ $: poetry add tox-poetry-installer --dev
```
2. Remove all version specifications from the environment dependencies in `tox.ini`:
Confirm that the plugin is installed, and Tox recognizes it, by checking the Tox version:
```
~ $: poetry run tox --version
3.20.0 imported from .venv/lib64/python3.8/site-packages/tox/__init__.py
registered plugins:
tox-poetry-installer-0.2.2 at .venv/lib64/python3.8/site-packages/tox_poetry_installer.py
```
If using Pip, ensure that the plugin is installed to the same environment as Tox:
```
# Calling the virtualenv's 'pip' binary directly will cause pip to install to that virtualenv
~ $: /path/to/my/automation/virtualenv/bin/pip install tox
~ $: /path/to/my/automation/virtualenv/bin/pip install tox-poetry-installer
```
## Quick Start
To add dependencies from the lockfile to a Tox environment, add the option `locked_deps`
to the environment configuration and list names of dependencies (with no version
specifier) under it:
```ini
# This...
[testenv]
description = My cool test environment
deps =
requests >=2.19,<3.0
toml == 0.10.0
pytest >=5.4
description = Some very cool tests
locked_deps =
black
pylint
mypy
commands = ...
```
# ...becomes this:
The standard `deps` option can be used in parallel with the `locked_deps` option to
install unlocked dependencies (dependencies not in the lockfile) alongside locked
dependencies:
```ini
[testenv]
description = My cool test environment
description = Some very cool tests
locked_deps =
black
pylint
mypy
deps =
pytest == 6.1.1
pytest-cov >= 2.10, <2.11
commands = ...
```
Alternatively, to quickly install all Poetry dev-dependencies to a Tox environment, add the
`install_dev_deps = true` option to the environment configuration.
**Note:** Regardless of the settings outlined above, all dependencies of the project package (the
one Tox is testing) will always be installed from the lockfile.
## Reference and Usage
### Config Option Reference
All options listed below are Tox environment options and can be applied to one or more
environment sections of the `tox.ini` file. They cannot be applied to the global Tox
configuration section.
**NOTE:** Environment settings applied to the main `testenv` environment will be
inherited by child environments (i.e. `testenv:foo`) unless they are explicitly
overridden by the child environment's configuration.
| Option | Type | Default | Usage |
|:----------------------|:----------------|:--------|:-----------------------------------------------|
| `locked_deps` | Multi-line list | `[]` | Names of packages in the Poetry lockfile to install to the Tox environment. All dependencies specified here (and their dependencies) will be installed to the Tox environment using the version the Poetry lockfile specifies for them. |
| `require_locked_deps` | Bool | `false` | Indicates whether the environment should allow unlocked dependencies (dependencies not in the Poetry lockfile) to be installed alongside locked dependencies. If `true` then installation of unlocked dependencies will be blocked and an error will be raised if the `deps` option specifies any values. |
| `install_dev_deps` | Bool | `false` | Indicates whether all Poetry development dependencies should be installed to the environment. Provides a quick and easy way to install all dev-dependencies without needing to specify them individually. |
### Error Reference
* `LockedDepVersionConflictError` - Indicates that a locked dependency included a PEP-508 version
specifier (i.e. `pytest >=6.0, <6.1`). Locked dependencies always take their version from the
Poetry lockfile so specifying a specific version for a locked dependency is not supported.
* `LockedDepNotFoundError` - Indicates that a locked dependency could not be found in the Poetry
lockfile. This can be solved by [adding the dependency using Poetry](https://python-poetry.org/docs/cli/#add).
* `ExtraNotFoundError` - Indicates that the Tox `extras` option specified a project extra that
Poetry does not know about. This may be due to a misconfigured `pyproject.toml` or out of date
lockfile.
* `LockedDepsRequiredError` - Indicates that an environment with `require_locked_deps = true` also
specified unlocked dependencies using Tox's `deps` option. This can be solved by either setting
`require_locked_deps = false` (the default) or removing the `deps` option from the environment
configuration.
### Example Config
```ini
[tox]
envlist = py, foo, bar, baz
isolated_build = true
# The base testenv will always use locked dependencies and only ever installs the project package
# (and its dependencies) and the two pytest dependencies listed below
[testenv]
description = Some very cool tests
require_locked_deps = true
locked_deps =
pytest
pytest-cov
commands = ...
# This environment also requires locked dependencies, but the "skip_install" setting means that
# the project dependencies will not be installed to the environment from the lockfile
[testenv:foo]
description = FOObarbaz
skip_install = true
require_locked_deps = true
locked_deps =
requests
toml
pytest
ruamel.yaml
commands = ...
# This environment allows unlocked dependencies to be installed ad-hoc. Below, the "mypy" and
# "pylint" dependencies (and their dependencies) will be installed from the Poetry lockfile but the
# "black" dependency will be installed using the default Tox backend. Note, this environment does
# not specify "require_locked_deps = true" to allow the unlocked "black" dependency without raising
# an error.
[testenv:bar]
description = fooBARbaz
locked_deps =
mypy
pylint
deps =
black
commands = ...
# This environment requires locked dependencies but does not specify any. Instead it specifies the
# "install_dev_deps = true" option which will cause all of the Poetry dev-dependencies to be
# installed from the lockfile.
[testenv:baz]
description = foobarBAZ
install_dev_deps = true
require_locked_deps = true
commands = ...
```
3. Run Tox with the `--recreate` flag to rebuild the test environments:
```
poetry run tox --recreate
```
## Known Drawbacks and Problems
4. 💸 Profit 💸
## Limitations
* In general, any command line or INI settings that affect how Tox installs environment
dependencies will be disabled by installing this plugin. A non-exhaustive and untested
list of the INI options that are not expected to work with this plugin is below:
* The following `tox.ini` configuration options have no effect on the dependencies installed from
the Poetry lockfile (note that they will still affect unlocked dependencies):
* [`install_command`](https://tox.readthedocs.io/en/latest/config.html#conf-install_command)
* [`pip_pre`](https://tox.readthedocs.io/en/latest/config.html#conf-pip_pre)
* [`downloadcache`](https://tox.readthedocs.io/en/latest/config.html#conf-downloadcache) (deprecated)
* [`download`](https://tox.readthedocs.io/en/latest/config.html#conf-download)
* [`indexserver`](https://tox.readthedocs.io/en/latest/config.html#conf-indexserver)
* [`usedevelop`](https://tox.readthedocs.io/en/latest/config.html#conf-indexserver)
* [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
* When the plugin is enabled all dependencies for all environments will use the Poetry backend
provided by the plugin; this functionality cannot be disabled on a per-environment basis.
* Tox will not automatically detect changes to the locked dependencies and so
environments will not be automatically rebuilt when locked dependencies are changed.
When changing the locked dependencies (or their versions) the environments will need to
be manually rebuilt using either the `-r`/`--recreate` CLI option or the
`recreate = true` option in `tox.ini`.
* Alternative versions cannot be specified alongside versions from the lockfile. All
dependencies are installed from the lockfile and alternative versions cannot be specified
in the Tox configuration.
* There are a handful of packages that cannot be installed from the lockfile, whether as specific
dependencies or as transient dependencies (dependencies of dependencies). This is due to
[an ongoing discussion in the Poetry project](https://github.com/python-poetry/poetry/issues/1584);
the list of dependencies that cannot be installed from the lockfile can be found
[here](https://github.com/python-poetry/poetry/blob/cc8f59a31567f806be868aba880ae0642d49b74e/poetry/puzzle/provider.py#L55).
This plugin will skip these dependencies entirely, but log a warning when they are encountered.
## Why would I use this?
**Introduction**
The lockfile is a file generated by a package manager for a project that lists what
dependencies are installed, the versions of those dependencies, and additional metadata that
the package manager can use to recreate the local project environment. This allows developers
The lockfile is a file generated by a package manager for a project that records what
dependencies are installed, the versions of those dependencies, and any additional metadata that
the package manager needs to recreate the local project environment. This allows developers
to have confidence that a bug they are encountering that may be caused by one of their
dependencies will be reproducible on another device. In addition, installing a project
environment from a lockfile gives confidence that automated systems running tests or performing
builds are using the same environment that a developer is.
builds are using the same environment as a developer.
[Poetry](https://python-poetry.org/) is a project dependency manager for Python projects, and
as such it creates and manages a lockfile so that its users can benefit from all the features
so it creates and manages a lockfile so that its users can benefit from all the features
described above. [Tox](https://tox.readthedocs.io/en/latest/#what-is-tox) is an automation tool
that allows Python developers to run tests suites, perform builds, and automate tasks within
self contained [Python virtual environments](https://docs.python.org/3/tutorial/venv.html).
To make these environments useful, Tox supports installing per-environment dependencies.
self-contained [Python virtual environments](https://docs.python.org/3/tutorial/venv.html).
To make these environments useful Tox supports installing dependencies in each environment.
However, since these environments are created on the fly and Tox does not maintain a lockfile,
there can be subtle differences between the dependencies a developer is using and the
dependencies Tox uses.
@@ -109,19 +238,17 @@ dependencies Tox uses.
This is where this plugin comes into play.
By default Tox uses [Pip](https://docs.python.org/3/tutorial/venv.html) to install the
PEP-508 compliant dependencies to a test environment. A more robust way to do this is to
install dependencies directly from the lockfile so that the version installed to the Tox
environment always matches the version Poetry specifies. This plugin overwrites the default
Tox dependency installation behavior and replaces it with a Poetry-based installation using
the dependency metadata from the lockfile.
PEP-508 compliant dependencies to a test environment. This plugin extends the default
Tox dependency installation behavior to support installing dependencies using a Poetry-based
installation method that makes use of the dependency metadata from Poetry's lockfile.
**The Problem**
Environment dependencies for a Tox environment are usually done in PEP-508 format like the
below example
Environment dependencies for a Tox environment are usually specified in PEP-508 format, like
the below example:
```ini
# tox.ini
# from tox.ini
...
[testenv]
@@ -134,31 +261,35 @@ deps =
...
```
Perhaps these dependencies are also useful during development, so they can be added to the
Let's assume these dependencies are also useful during development, so they can be added to the
Poetry environment using this command:
```
poetry add foo==1.2.3 bar>=1.3,<2.0 baz --dev
poetry add --dev \
foo==1.2.3 \
bar>=1.3,<2.0 \
baz
```
However there are three potential problems that could arise from each of these environment
However there is a potential problem that could arise from each of these environment
dependencies that would _only_ appear in the Tox environment and not in the Poetry
environment:
environment in use by a developer:
* **The `foo` dependency is pinned to a specific version:** let's imagine a security
vulnerability is discovered in `foo` and the maintainers release version `1.2.4` to fix
it. A developer can run `poetry remove foo && poetry add foo^1.2` to get the new version,
but the Tox environment is left unchanged. The developer environment specified by the
lockfile is now patched against the vulnerability, but the Tox environment is not.
it. A developer can run `poetry remove foo` and then `poetry add foo^1.2` to get the new
version, but the Tox environment is left unchanged. The development environment, as defined by
the lockfile, is now patched against the vulnerability but the Tox environment is not.
* **The `bar` dependency specifies a dynamic range:** a dynamic range allows a range of
versions to be installed, but the lockfile will have an exact version specified so that
the Poetry environment is reproducible; this allows versions to be updated with
`poetry update` rather than with the `remove` and `add` used above. If the maintainers of
`bar` release version `1.6.0` then the Tox environment will install it because it is valid
for the specified version range, meanwhile the Poetry environment will continue to install
the version from the lockfile until `poetry update bar` explicitly updates it. The
development environment is now has a different version of `bar` than the Tox environment.
`poetry update` rather than with the `remove` and `add` commands used above. If the
maintainers of `bar` release version `1.6.0` then the Tox environment will install it
because it is valid for the specified version range. Meanwhile the Poetry environment will
continue to install the version from the lockfile until `poetry update bar` explicitly
updates it. The development environment is now has a different version of `bar` than the Tox
environment.
* **The `baz` dependency is unpinned:** unpinned dependencies are
[generally a bad idea](https://python-poetry.org/docs/faq/#why-are-unbound-version-constraints-a-bad-idea),
@@ -167,28 +298,27 @@ Poetry environment using this command:
but Pip (via Tox) will interpret it as a wildcard. If the latest version of `baz` is `1.0.0`
then `poetry add baz` will result in a constraint of `baz>=1.0.0,<2.0.0` while the Tox
environment will have a constraint of `baz==*`. The Tox environment can now install an
incompatible version of `baz` that cannot be easily caught using `poetry update`.
incompatible version of `baz` and any errors that causes cannot be replicated using `poetry update`.
All of these problems can apply not only to the dependencies specified for a Tox environment,
but also to the dependencies of those dependencies, and so on.
but also to the dependencies of those dependencies, those dependencies' dependencies, and so on.
**The Solution**
This plugin requires that all dependencies specified for all Tox environments be unbound
with no version constraint specified at all. This seems counter-intuitive given the problems
outlined above, but what it allows the plugin to do is offload all version management to
Poetry.
On initial inspection, the environment below appears less stable than the one presented above
because it does not specify any versions for its dependencies:
This plugin allows dependencies specified in Tox environment take their version directly from
the Poetry lockfile without needing an independent version to be specified in the Tox
environment configuration. The modified version of the example environment given below appears
less stable than the one presented above because it does not specify any versions for its
dependencies:
```ini
# tox.ini
# from tox.ini
...
[testenv]
description = Some very cool tests
deps =
require_locked_deps = true
locked_deps =
foo
bar
baz
@@ -196,19 +326,17 @@ deps =
...
```
However with the `tox-poetry-installer` plugin installed this instructs Tox to install these
dependencies using the Poetry lockfile so that the version installed to the Tox environment
exactly matches the version Poetry is managing. When `poetry update` updates the lockfile
with new dependency versions, Tox will automatically install these new versions without needing
any changes to the configuration.
All dependencies are specified in one place (the lockfile) and dependency version management is
handled by a tool dedicated to that task (Poetry).
However with the `tox-poetry-installer` plugin installed the `require_locked_deps = true`
setting means that Tox will install these dependencies from the Poetry lockfile so that the
version installed to the Tox environment exactly matches the version Poetry is managing. When
`poetry update` updates the lockfile with new versions of these dependencies, Tox will
automatically install these new versions without needing any changes to the configuration.
## Developing
This project requires Poetry-1.0+, see the [installation instructions here](https://python-poetry.org/docs/#installation).
This project requires a developer to have Poetry version 1.0+ installed on their workstation, see
the [installation instructions here](https://python-poetry.org/docs/#installation).
```bash
# Clone the repository...
@@ -234,6 +362,10 @@ poetry run tox
All project contributors and participants are expected to adhere to the
[Contributor Covenant Code of Conduct, Version 2](CODE_OF_CONDUCT.md).
The `devel` branch has the latest (potentially unstable) changes. The
[tagged versions](https://github.com/enpaul/tox-poetry-installer/releases) correspond to the
releases on PyPI.
* To report a bug, request a feature, or ask for assistance, please
[open an issue on the Github repository](https://github.com/enpaul/tox-poetry-installer/issues/new).
* To report a security concern or code of conduct violation, please contact the project author
@@ -247,7 +379,7 @@ All project contributors and participants are expected to adhere to the
## Roadmap
This project is under active development and is classified as alpha software, not yet ready
usage in production systems.
for usage in production environments.
* Beta classification will be assigned when the initial feature set is finalized
* Stable classification will be assigned when the test suite covers an acceptable number of
@@ -255,21 +387,23 @@ usage in production systems.
### Path to Beta
- [ ] Verify that primary package dependencies (from the `.package` env) are installed
- [X] Verify that primary package dependencies (from the `.package` env) are installed
correctly using the Poetry backend.
- [ ] Support the [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
Tox configuration option
- [ ] Add per-environment Tox configuration option to fall back to default installation
- [X] Support the [`extras`](https://tox.readthedocs.io/en/latest/config.html#conf-extras)
Tox configuration option ([#4](https://github.com/enpaul/tox-poetry-installer/issues/4))
- [X] Add per-environment Tox configuration option to fall back to default installation
backend.
- [ ] Add detection of a changed lockfile to automatically trigger a rebuild of Tox
environments when necessary.
- [ ] Add warnings when an unsupported Tox configuration option is detected while using the
Poetry backend.
Poetry backend. ([#5](https://github.com/enpaul/tox-poetry-installer/issues/5))
- [X] Add trivial tests to ensure the project metadata is consistent between the pyproject.toml
and the module constants.
- [ ] Update to use [poetry-core](https://github.com/python-poetry/poetry-core)
Tox configuration option) and improve robustness of the Tox and Poetry module imports
to avoid potentially breaking API changes in upstream packages.
- [X] Update to use [poetry-core](https://github.com/python-poetry/poetry-core) and
improve robustness of the Tox and Poetry module imports
to avoid potentially breaking API changes in upstream packages. ([#2](https://github.com/enpaul/tox-poetry-installer/issues/2))
- [ ] Find and implement a way to mitigate the [UNSAFE_DEPENDENCIES issue](https://github.com/python-poetry/poetry/issues/1584) in Poetry.
([#6](https://github.com/enpaul/tox-poetry-installer/issues/6))
- [ ] Fix logging to make proper use of Tox's logging reporter infrastructure ([#3](https://github.com/enpaul/tox-poetry-installer/issues/3))
- [X] Add configuration option for installing all dev-dependencies to a testenv ([#14](https://github.com/enpaul/tox-poetry-installer/issues/14))
### Path to Stable
@@ -277,6 +411,6 @@ Everything in Beta plus...
- [ ] Add tests for each feature version of Tox between 2.3 and 3.20
- [ ] Add tests for Python-3.6, 3.7, and 3.8
- [ ] Add Github Actions based CI
- [X] Add Github Actions based CI
- [ ] Add CI for CPython, PyPy, and Conda
- [ ] Add CI for Linux and Windows

1039
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,11 +1,17 @@
[tool.poetry]
name = "tox-poetry-installer"
version = "0.1.2"
version = "0.5.0"
license = "MIT"
authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
description = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
repository = "https://github.com/enpaul/tox-poetry-installer/"
packages = [{include = "tox_poetry_installer.py"}]
packages = [
{include = "tox_poetry_installer"},
{include = "tests/*.py", format = "sdist"}
]
include = [
"tox_poetry_installer/py.typed"
]
keywords = ["tox", "poetry", "plugin"]
readme = "README.md"
classifiers = [
@@ -28,6 +34,7 @@ poetry_installer = "tox_poetry_installer"
[tool.poetry.dependencies]
python = "^3.6"
poetry = "^1.0.0"
poetry-core = "^1.0.0"
tox = "^2.3.0 || ^3.0.0"
[tool.poetry.dev-dependencies]
@@ -45,5 +52,5 @@ toml = "^0.10.1"
tox = "^3.20.0"
[build-system]
requires = ["poetry>=1.0.0"]
build-backend = "poetry.masonry.api"
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@@ -7,7 +7,7 @@ from pathlib import Path
import toml
import tox_poetry_installer
from tox_poetry_installer import __about__
def test_metadata():
@@ -16,16 +16,14 @@ def test_metadata():
with (Path(__file__).resolve().parent / ".." / "pyproject.toml").open() as infile:
pyproject = toml.load(infile, _dict=dict)
assert pyproject["tool"]["poetry"]["name"] == tox_poetry_installer.__title__
assert pyproject["tool"]["poetry"]["version"] == tox_poetry_installer.__version__
assert pyproject["tool"]["poetry"]["license"] == tox_poetry_installer.__license__
assert (
pyproject["tool"]["poetry"]["description"] == tox_poetry_installer.__summary__
)
assert pyproject["tool"]["poetry"]["repository"] == tox_poetry_installer.__url__
assert pyproject["tool"]["poetry"]["name"] == __about__.__title__
assert pyproject["tool"]["poetry"]["version"] == __about__.__version__
assert pyproject["tool"]["poetry"]["license"] == __about__.__license__
assert pyproject["tool"]["poetry"]["description"] == __about__.__summary__
assert pyproject["tool"]["poetry"]["repository"] == __about__.__url__
assert (
all(
item in tox_poetry_installer.__authors__
item in __about__.__authors__
for item in pyproject["tool"]["poetry"]["authors"]
)
is True
@@ -33,7 +31,7 @@ def test_metadata():
assert (
all(
item in pyproject["tool"]["poetry"]["authors"]
for item in tox_poetry_installer.__authors__
for item in __about__.__authors__
)
is True
)

54
tox.ini
View File

@@ -4,35 +4,49 @@ isolated_build = true
[testenv]
description = Run the tests
deps =
require_locked_deps = true
locked_deps =
pytest
pytest-cov
toml
commands =
pytest --cov tox_poetry_installer --cov-config {toxinidir}/.coveragerc tests/ --cov-report term-missing
pytest --cov {envsitepackagesdir}/tox_poetry_installer --cov-config {toxinidir}/.coveragerc --cov-report term-missing tests/
[testenv:static]
description = Static formatting and quality enforcement
basepython = python3.8
platform = linux
ignore_errors = true
deps =
require_locked_deps = true
locked_deps =
pylint
mypy
black
reorder-python-imports
pre-commit
allowlist_externals =
bash
commands =
black {toxinidir}/tox_poetry_installer.py
reorder-python-imports {toxinidir}/tox_poetry_installer.py
black {toxinidir}/tox_poetry_installer/
# Oh man this is a doozy. If submodules are ever added to this plugin this will break, but I'm
# frustrated enough at this point that I'll need to take another look at it later to fix that.
# reorder-python-imports doesn't support handling directories on the CLI
# (https://github.com/asottile/reorder_python_imports/pull/76) and because the command is
# invoked directly (see comment below) we need file globbing to work around it.
# The "--unclassifiable-application-module" is a work around for reorder-python-imports not
# properly detecting the top-level module when run in a bash-wrapped command like this.
bash -c "reorder-python-imports {toxinidir}/tox_poetry_installer/*.py --unclassifiable-application-module tox_poetry_installer"
pre-commit run --all-files
pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tox_poetry_installer.py
mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tox_poetry_installer.py
pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tox_poetry_installer/
mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tox_poetry_installer/
[testenv:static-tests]
description = Static formatting and quality enforcement for the tests
basepython = python3.8
platform = linux
ingore_errors = true
deps =
require_locked_deps = true
locked_deps =
pylint
mypy
black
@@ -41,22 +55,26 @@ allowlist_externals =
bash
commands =
black {toxinidir}/tests/
# These bash-wrapped commands hurt my face, but these tools expect directories to be valid
# python modules, which the "tests/" directory is not. Since tox calls all commands directly
# (which is good) file globbing doesn't work. To make file globbing work they need to be wrapped
# in a bash call (which is bad).
bash -c "reorder-python-imports {toxinidir}/tests/*.py --unclassifiable-application-module tox_poetry_installer"
pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tests/
mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tests/
bash -c "pylint --rcfile {toxinidir}/.pylintrc {toxinidir}/tests/*.py"
bash -c "mypy --ignore-missing-imports --no-strict-optional {toxinidir}/tests/*.py"
[testenv:security]
description = Security checks
basepython = python3.8
ignore_errors = true
skip_install = true
deps =
platform = linux
ingore_errors = true
require_locked_deps = true
locked_deps =
bandit
safety
poetry
allowlist_externals =
bash
commands =
bandit --recursive {toxinidir}/tox_poetry_installer.py
bash -c "bandit --skip B101 {toxinidir}/tests/*.py"
bash -c "poetry export --format requirements.txt --without-hashes --dev | safety check --stdin --bare"
bandit --recursive --quiet {toxinidir}/tox_poetry_installer/
bandit --recursive --quiet --skip B101 {toxinidir}/tests/
poetry export --format requirements.txt --output {envtmpdir}/requirements.txt --without-hashes --dev
safety check --bare --file {envtmpdir}/requirements.txt

View File

@@ -1,142 +0,0 @@
"""Tox plugin for installing environments using Poetry
This plugin makes use of the ``tox_testenv_install_deps`` Tox plugin hook to replace the default
installation functionality to install dependencies from the Poetry lockfile for the project. It
does this by using ``poetry`` to read in the lockfile, identify necessary dependencies, and then
use Poetry's ``PipInstaller`` class to install those packages into the Tox environment.
"""
import logging
from pathlib import Path
from typing import Dict
from typing import List
from typing import Optional
from typing import Tuple
from poetry.factory import Factory as PoetryFactory
from poetry.factory import Poetry
from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
from poetry.io.null_io import NullIO as PoetryNullIO
from poetry.packages import Package as PoetryPackage
from poetry.puzzle.provider import Provider as PoetryProvider
from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
from tox import hookimpl
from tox.action import Action as ToxAction
from tox.venv import VirtualEnv as ToxVirtualEnv
__title__ = "tox-poetry-installer"
__summary__ = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
__version__ = "0.1.2"
__url__ = "https://github.com/enpaul/tox-poetry-installer/"
__license__ = "MIT"
__authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
_PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<")
class ToxPoetryInstallerException(Exception):
"""Error while installing locked dependencies to the test environment"""
class NoLockedDependencyError(ToxPoetryInstallerException):
"""Cannot install a package that is not in the lockfile"""
def _make_poetry(venv: ToxVirtualEnv) -> Poetry:
"""Helper to make a poetry object from a toxenv"""
return PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
def _find_locked_dependencies(
poetry: Poetry, dependency_name: str
) -> List[PoetryPackage]:
"""Using a poetry object identify all dependencies of a specific dependency
:param poetry: Populated poetry object which can be used to build a populated locked
repository object.
:param dependency_name: Bare name (without version) of the dependency to fetch the transient
dependencies of.
:returns: List of packages that need to be installed for the requested dependency.
.. note:: The package corresponding to the dependency named by ``dependency_name`` is included
in the list of returned packages.
"""
packages: Dict[str, PoetryPackage] = {
package.name: package
for package in poetry.locker.locked_repository(True).packages
}
try:
top_level = packages[dependency_name]
def find_transients(name: str) -> List[PoetryPackage]:
if name in PoetryProvider.UNSAFE_PACKAGES:
return []
transients = [packages[name]]
for dep in packages[name].requires:
transients += find_transients(dep.name)
return transients
return find_transients(top_level.name)
except KeyError:
if any(
delimiter in dependency_name for delimiter in _PEP508_VERSION_DELIMITERS
):
message = "specifying a version in the tox environment definition is incompatible with installing from a lockfile"
else:
message = (
"no version of the package was found in the current project's lockfile"
)
raise NoLockedDependencyError(
f"Cannot install requirement '{dependency_name}': {message}"
) from None
@hookimpl
def tox_testenv_install_deps(
venv: ToxVirtualEnv, action: ToxAction
) -> Optional[List[PoetryPackage]]:
"""Install the dependencies for the current environment
Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
specified by the Tox environment. Finally these dependencies are installed into the Tox
environment using the Poetry ``PipInstaller`` backend.
:param venv: Tox virtual environment object with configuration for the local Tox environment.
:param action: Tox action object
"""
logger = logging.getLogger(__name__)
if action.name == venv.envconfig.config.isolated_build_env:
logger.debug(
f"Environment {action.name} is isolated build environment; skipping Poetry-based dependency installation"
)
return None
poetry = _make_poetry(venv)
logger.debug(f"Loaded project pyproject.toml from {poetry.file}")
dependencies: List[PoetryPackage] = []
for env_dependency in venv.envconfig.deps:
dependencies += _find_locked_dependencies(poetry, env_dependency.name)
logger.debug(
f"Identified {len(dependencies)} dependencies for environment {action.name}"
)
installer = PoetryPipInstaller(
env=PoetryVirtualEnv(path=Path(venv.envconfig.envdir)),
io=PoetryNullIO(),
pool=poetry.pool,
)
for dependency in dependencies:
logger.info(f"Installing environment dependency: {dependency}")
installer.install(dependency)
return dependencies

View File

@@ -0,0 +1,7 @@
# pylint: disable=missing-docstring
__title__ = "tox-poetry-installer"
__summary__ = "Tox plugin to install Tox environment dependencies using the Poetry backend and lockfile"
__version__ = "0.5.0"
__url__ = "https://github.com/enpaul/tox-poetry-installer/"
__license__ = "MIT"
__authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]

View File

@@ -0,0 +1,3 @@
# pylint: disable=missing-docstring
from tox_poetry_installer.hooks import tox_addoption
from tox_poetry_installer.hooks import tox_testenv_install_deps

View File

@@ -0,0 +1,19 @@
"""Static constants for reference
Rule of thumb: if it's an arbitrary value that will never be changed at runtime, it should go
in this module.
All constants should be type hinted.
"""
from typing import Tuple
from tox_poetry_installer import __about__
# Valid PEP508 version delimiters. These are used to test whether a given string (specifically a
# dependency name) is just a package name or also includes a version identifier.
PEP508_VERSION_DELIMITERS: Tuple[str, ...] = ("~=", "==", "!=", ">", "<")
# Prefix all reporter messages should include to indicate that they came from this module in the
# console output.
REPORTER_PREFIX = f"[{__about__.__title__}]:"

View File

@@ -0,0 +1,8 @@
"""Definitions for typehints/containers used by the plugin"""
from typing import Dict
from poetry.core.packages import Package as PoetryPackage
# Map of package names to the package object
PackageMap = Dict[str, PoetryPackage]

View File

@@ -0,0 +1,33 @@
"""Custom plugin exceptions
All exceptions should inherit from the common base exception :exc:`ToxPoetryInstallerException`.
::
ToxPoetryInstallerException
+-- LockedDepVersionConflictError
+-- LockedDepNotFoundError
+-- ExtraNotFoundError
+-- LockedDepsRequiredError
"""
class ToxPoetryInstallerException(Exception):
"""Error while installing locked dependencies to the test environment"""
class LockedDepVersionConflictError(ToxPoetryInstallerException):
"""Locked dependencies cannot specify an alternate version for installation"""
class LockedDepNotFoundError(ToxPoetryInstallerException):
"""Locked dependency was not found in the lockfile"""
class ExtraNotFoundError(ToxPoetryInstallerException):
"""Project package extra not defined in project's pyproject.toml"""
class LockedDepsRequiredError(ToxPoetryInstallerException):
"""Environment cannot specify unlocked dependencies when locked dependencies are required"""

View File

@@ -0,0 +1,221 @@
"""Main hook definition module
All implementations of tox hooks are defined here, as well as any single-use helper functions
specifically related to implementing the hooks (to keep the size/readability of the hook functions
themselves manageable).
"""
from typing import List
from typing import Optional
from poetry.core.packages import Package as PoetryPackage
from poetry.factory import Factory as PoetryFactory
from poetry.poetry import Poetry
from tox import hookimpl
from tox import reporter
from tox.action import Action as ToxAction
from tox.config import Parser as ToxParser
from tox.venv import VirtualEnv as ToxVirtualEnv
from tox_poetry_installer import constants
from tox_poetry_installer import exceptions
from tox_poetry_installer import utilities
from tox_poetry_installer.datatypes import PackageMap
@hookimpl
def tox_addoption(parser: ToxParser):
"""Add required configuration options to the tox INI file
Adds the ``require_locked_deps`` configuration option to the venv to check whether all
dependencies should be treated as locked or not.
"""
parser.add_testenv_attribute(
name="install_dev_deps",
type="bool",
default=False,
help="Automatically install all Poetry development dependencies to the environment",
)
parser.add_testenv_attribute(
name="require_locked_deps",
type="bool",
default=False,
help="Require all dependencies in the environment be installed using the Poetry lockfile",
)
parser.add_testenv_attribute(
name="locked_deps",
type="line-list",
help="List of locked dependencies to install to the environment using the Poetry lockfile",
)
@hookimpl
def tox_testenv_install_deps(venv: ToxVirtualEnv, action: ToxAction) -> Optional[bool]:
"""Install the dependencies for the current environment
Loads the local Poetry environment and the corresponding lockfile then pulls the dependencies
specified by the Tox environment. Finally these dependencies are installed into the Tox
environment using the Poetry ``PipInstaller`` backend.
:param venv: Tox virtual environment object with configuration for the local Tox environment.
:param action: Tox action object
"""
if action.name == venv.envconfig.config.isolated_build_env:
# Skip running the plugin for the packaging environment. PEP-517 front ends can handle
# that better than we can, so let them do their thing. More to the point: if you're having
# problems in the packaging env that this plugin would solve, god help you.
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} skipping isolated build env '{action.name}'"
)
return None
try:
poetry = PoetryFactory().create_poetry(venv.envconfig.config.toxinidir)
except RuntimeError:
# Support running the plugin when the current tox project does not use Poetry for its
# environment/dependency management.
#
# ``RuntimeError`` is dangerous to blindly catch because it can be (and in Poetry's case,
# is) raised in many different places for different purposes.
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} project does not use Poetry for env management, skipping installation of locked dependencies"
)
return None
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} loaded project pyproject.toml from {poetry.file}"
)
package_map: PackageMap = {
package.name: package
for package in poetry.locker.locked_repository(True).packages
}
if venv.envconfig.require_locked_deps and venv.envconfig.deps:
raise exceptions.LockedDepsRequiredError(
f"Unlocked dependencies '{venv.envconfig.deps}' specified for environment '{venv.name}' which requires locked dependencies"
)
# Handle the installation of any locked env dependencies from the lockfile
_install_env_dependencies(venv, poetry, package_map)
# Handle the installation of the package dependencies from the lockfile if the package is
# being installed to this venv; otherwise skip installing the package dependencies
if venv.envconfig.skip_install:
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} env specifies 'skip_install = true', skipping installation of project package"
)
return venv.envconfig.require_locked_deps or None
if venv.envconfig.config.skipsdist:
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} config specifies 'skipsdist = true', skipping installation of project package"
)
return venv.envconfig.require_locked_deps or None
_install_project_dependencies(venv, poetry, package_map)
return venv.envconfig.require_locked_deps or None
def _install_env_dependencies(
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
):
"""Install the packages for a specified testenv
Processes the tox environment config, identifies any locked environment dependencies, pulls
them from the lockfile, and installs them to the virtual environment.
:param venv: Tox virtual environment to install the packages to
:param poetry: Poetry object the packages were sourced from
:param packages: Mapping of package names to the corresponding package object
"""
dependencies: List[PoetryPackage] = []
for dep in venv.envconfig.locked_deps:
try:
dependencies += utilities.find_transients(packages, dep.lower())
except exceptions.ToxPoetryInstallerException as err:
venv.status = "lockfile installation failed"
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
raise err
if venv.envconfig.install_dev_deps:
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} env specifies 'install_env_deps = true', including Poetry dev dependencies"
)
dev_dependencies = [
dep
for dep in poetry.locker.locked_repository(True).packages
if dep not in poetry.locker.locked_repository(False).packages
]
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} identified {len(dev_dependencies)} Poetry dev dependencies"
)
dependencies = list(set(dev_dependencies + dependencies))
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(venv.envconfig.locked_deps)} locked env dependencies"
)
reporter.verbosity0(
f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} env dependencies from lockfile"
)
utilities.install_to_venv(poetry, venv, dependencies)
def _install_project_dependencies(
venv: ToxVirtualEnv, poetry: Poetry, packages: PackageMap
):
"""Install the dependencies of the project package
Install all primary dependencies of the project package.
:param venv: Tox virtual environment to install the packages to
:param poetry: Poetry object the packages were sourced from
:param packages: Mapping of package names to the corresponding package object
"""
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} performing installation of project dependencies"
)
base_dependencies: List[PoetryPackage] = [
packages[item.name]
for item in poetry.package.requires
if not item.is_optional()
]
extra_dependencies: List[PoetryPackage] = []
for extra in venv.envconfig.extras:
try:
extra_dependencies += [
packages[item.name] for item in poetry.package.extras[extra]
]
except KeyError:
raise exceptions.ExtraNotFoundError(
f"Environment '{venv.name}' specifies project extra '{extra}' which was not found in the lockfile"
) from None
dependencies: List[PoetryPackage] = []
for dep in base_dependencies + extra_dependencies:
try:
dependencies += utilities.find_transients(packages, dep.name.lower())
except exceptions.ToxPoetryInstallerException as err:
venv.status = "lockfile installation failed"
reporter.error(f"{constants.REPORTER_PREFIX} {err}")
raise err
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} identified {len(dependencies)} total dependencies from {len(poetry.package.requires)} project dependencies"
)
reporter.verbosity0(
f"{constants.REPORTER_PREFIX} ({venv.name}) installing {len(dependencies)} project dependencies from lockfile"
)
utilities.install_to_venv(poetry, venv, dependencies)

View File

View File

@@ -0,0 +1,85 @@
"""Helper utility functions, usually bridging Tox and Poetry functionality"""
from pathlib import Path
from typing import Sequence
from typing import Set
from poetry.core.packages import Package as PoetryPackage
from poetry.installation.pip_installer import PipInstaller as PoetryPipInstaller
from poetry.io.null_io import NullIO as PoetryNullIO
from poetry.poetry import Poetry
from poetry.puzzle.provider import Provider as PoetryProvider
from poetry.utils.env import VirtualEnv as PoetryVirtualEnv
from tox import reporter
from tox.venv import VirtualEnv as ToxVirtualEnv
from tox_poetry_installer import constants
from tox_poetry_installer import exceptions
from tox_poetry_installer.datatypes import PackageMap
def install_to_venv(
poetry: Poetry, venv: ToxVirtualEnv, packages: Sequence[PoetryPackage]
):
"""Install a bunch of packages to a virtualenv
:param poetry: Poetry object the packages were sourced from
:param venv: Tox virtual environment to install the packages to
:param packages: List of packages to install to the virtual environment
"""
reporter.verbosity1(
f"{constants.REPORTER_PREFIX} Installing {len(packages)} packages to environment at {venv.envconfig.envdir}"
)
installer = PoetryPipInstaller(
env=PoetryVirtualEnv(path=Path(venv.envconfig.envdir)),
io=PoetryNullIO(),
pool=poetry.pool,
)
for dependency in packages:
reporter.verbosity1(f"{constants.REPORTER_PREFIX} installing {dependency}")
installer.install(dependency)
def find_transients(packages: PackageMap, dependency_name: str) -> Set[PoetryPackage]:
"""Using a poetry object identify all dependencies of a specific dependency
:param poetry: Populated poetry object which can be used to build a populated locked
repository object.
:param dependency_name: Bare name (without version) of the dependency to fetch the transient
dependencies of.
:returns: List of packages that need to be installed for the requested dependency.
.. note:: The package corresponding to the dependency named by ``dependency_name`` is included
in the list of returned packages.
"""
try:
def find_deps_of_deps(name: str, transients: PackageMap):
if name in PoetryProvider.UNSAFE_PACKAGES:
reporter.warning(
f"{constants.REPORTER_PREFIX} installing package '{name}' using Poetry is not supported; skipping installation of package '{name}'"
)
else:
transients[name] = packages[name]
for dep in packages[name].requires:
if dep.name not in transients.keys():
find_deps_of_deps(dep.name, transients)
transients: PackageMap = {}
find_deps_of_deps(packages[dependency_name].name, transients)
return set(transients.values())
except KeyError:
if any(
delimiter in dependency_name
for delimiter in constants.PEP508_VERSION_DELIMITERS
):
raise exceptions.LockedDepVersionConflictError(
f"Locked dependency '{dependency_name}' cannot include version specifier"
) from None
raise exceptions.LockedDepNotFoundError(
f"No version of locked dependency '{dependency_name}' found in the project lockfile"
) from None