.github/scripts/setup-env.sh

@@ -4,34 +4,31 @@
 # to create a repeatable local environment for tests to be run in. The python env
 # this script creates can be accessed at the location defined by the CI_VENV variable
 # below.
-#
-# POETRY_VERSION can be set to install a specific version of Poetry
 
 set -e;
 
 CI_CACHE=$HOME/.cache;
-INSTALL_POETRY_VERSION="${POETRY_VERSION:-1.3.2}";
+POETRY_VERSION=1.1.12;
 
 mkdir --parents "$CI_CACHE";
 
 command -v python;
-python3.10 --version;
+python --version;
 
 curl --location https://install.python-poetry.org \
   --output "$CI_CACHE/install-poetry.py" \
   --silent \
   --show-error;
 python "$CI_CACHE/install-poetry.py" \
-  --version "$INSTALL_POETRY_VERSION" \
+  --version "$POETRY_VERSION" \
   --yes;
 poetry --version --no-ansi;
 poetry run pip --version;
 
 poetry install \
-  --sync \
-  --no-ansi \
-  --no-root \
-  --only ci;
+  --quiet \
+  --remove-untracked \
+  --no-ansi;
 
 poetry env info;
 poetry run tox --version;

.github/workflows/ci.yaml

@@ -5,16 +5,14 @@ on:
     types: ["opened", "synchronize"]
   push:
     branches: ["devel"]
-env:
-  POETRY_VERSION: 1.4.1
 jobs:
   Test:
-    name: Python ${{ matrix.python.version }}
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: true
       matrix:
         python:
+          - version: "3.6"
+            toxenv: py36
           - version: "3.7"
             toxenv: py37
           - version: "3.8"
@@ -23,24 +21,15 @@ jobs:
             toxenv: py39
           - version: "3.10"
             toxenv: py310
-          - version: "3.11"
-            toxenv: py311
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
-      - name: Install Python 3.10
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-
       - name: Install Python ${{ matrix.python.version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python.version }}
-
       - name: Configure Job Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: |
             ~/.cache/pip
@@ -50,49 +39,38 @@ jobs:
           # will be invalidated, and thus all packages will be redownloaded, if the
           # lockfile is updated
           key: ${{ runner.os }}-${{ matrix.python.toxenv }}-${{ hashFiles('**/poetry.lock') }}
-
       - name: Configure Path
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-
       - name: Configure Environment
         run: .github/scripts/setup-env.sh
-
       - name: Run Toxenv ${{ matrix.python.toxenv }}
         run: poetry run tox -e ${{ matrix.python.toxenv }}
-
   Check:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-
-      - name: Install Python 3.10
-        uses: actions/setup-python@v4
+      - name: Install Python 3.8
+        uses: actions/setup-python@v1
         with:
-          python-version: "3.10"
-
+          python-version: 3.8
       - name: Configure Job Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: |
             ~/.cache/pip
             ~/.cache/pypoetry/cache
             ~/.poetry
-          # Hardcoded 'py310' slug here lets this cache piggyback on the 'py310' cache
+          # Hardcoded 'py38' slug here lets this cache piggyback on the 'py38' cache
           # that is generated for the tests above
-          key: ${{ runner.os }}-py310-${{ hashFiles('**/poetry.lock') }}
-
+          key: ${{ runner.os }}-py38-${{ hashFiles('**/poetry.lock') }}
       - name: Configure Path
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
-
       - name: Configure Environment
         run: .github/scripts/setup-env.sh
-
       - name: Run Static Analysis Checks
         run: poetry run tox -e static
-
       - name: Run Static Analysis Checks (Tests)
         run: poetry run tox -e static-tests
-
       - name: Run Security Checks
         run: poetry run tox -e security

.pylintrc

@@ -11,6 +11,7 @@
 # --disable=W"
 disable=logging-fstring-interpolation
         ,logging-format-interpolation
+        ,bad-continuation
         ,line-too-long
         ,ungrouped-imports
         ,typecheck

CHANGELOG.md

@@ -2,23 +2,6 @@
 
 See also: [Github Release Page](https://github.com/enpaul/vault2vault/releases).
 
-## Version 0.1.3
-
-View this release on: [Github](https://github.com/enpaul/vault2vault/releases/tag/0.1.3),
-[PyPI](https://pypi.org/project/vault2vault/0.1.3/)
-
-- Fix incorrect encoding specification when opening password files. Contributed by
-  [brycelowe](https://github.com/brycelowe) (#2)
-
-## Version 0.1.2
-
-View this release on: [Github](https://github.com/enpaul/vault2vault/releases/tag/0.1.2),
-[PyPI](https://pypi.org/project/vault2vault/0.1.2/)
-
-- Add user documentation
-- Add project road map
-- Fix incorrect and missing docstrings for internal functions
-
 ## Version 0.1.1
 
 View this release on: [Github](https://github.com/enpaul/vault2vault/releases/tag/0.1.1),
@@ -26,8 +9,8 @@ View this release on: [Github](https://github.com/enpaul/vault2vault/releases/ta
 
 - Fix bug causing stack trace when the same vaulted block appears in a YAML file more than
   once
-- Fix bug where the `--ignore-undecryptable` option was not respected for vaulted
-  variables in YAML files
+- Fix bug where the `--ignore-undecryptable` option was not respected for vaulted variables
+  in YAML files
 - Update logging messages and levels to improve verbose output
 
 ## Version 0.1.0

CODE_OF_CONDUCT.md

@@ -27,10 +27,9 @@ Examples of unacceptable behavior include:
 - The use of sexualized language or imagery, and sexual attention or advances of any kind
 - Trolling, insulting or derogatory comments, and personal or political attacks
 - Public or private harassment
-- Publishing others' private information, such as a physical or email address, without
-  their explicit permission
-- Other conduct which could reasonably be considered inappropriate in a professional
-  setting
+- Publishing others' private information, such as a physical or email address, without their
+  explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
 
 ## Enforcement Responsibilities
 
@@ -53,8 +52,8 @@ offline event.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the
-community leaders responsible for enforcement at \[INSERT CONTACT METHOD\]. All complaints
-will be reviewed and investigated promptly and fairly.
+community leaders responsible for enforcement at \[INSERT CONTACT METHOD\]. All
+complaints will be reviewed and investigated promptly and fairly.
 
 All community leaders are obligated to respect the privacy and security of the reporter of
 any incident.
@@ -106,8 +105,8 @@ toward or disparagement of classes of individuals.
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0,
 available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
 
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity).
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
 
 For answers to common questions about this code of conduct, see the FAQ at
 https://www.contributor-covenant.org/faq. Translations are available at

Makefile

@@ -30,10 +30,10 @@ source: ## Build Python source distribution package
 	poetry build --format sdist
 
 test: ## Run the project testsuite(s)
-	poetry run tox --recreate --parallel
+	poetry run tox --recreate
 
 dev: ## Create the local dev environment
-	poetry install --with dev --extras ansible --sync
+	poetry install
 	poetry run pre-commit install
 
 publish: test wheel source ## Build and upload to pypi (requires $PYPI_API_KEY be set)

README.md

@@ -10,13 +10,11 @@ but works recursively on encrypted files and in-line variables
 [![Python Supported Versions](https://img.shields.io/pypi/pyversions/vault2vault)](https://www.python.org)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
-⚠️ **This project is beta software and is under active development** ⚠️
+⚠️ **This project is alpha software and is under active development** ⚠️
 
 - [What is this?](#what-is-this)
 - [Installing](#installing)
-- [Usage](#usage)
-  - [Recovering from a failed migration](#recovering-from-a-failed-migration)
-- [Roadmap](#roadmap)
+- [Using](#using)
 - [Developing](#developer-documentation)
 
 ## What is this?
@@ -29,10 +27,13 @@ terminal. Whatever, these things happen.
 
 The built-in tool Ansible provides,
 [`ansible-vault rekey`](https://docs.ansible.com/ansible/latest/cli/ansible-vault.html#rekey),
-suffers from two main drawbacks: first, it only works on vault encrypted files and not on
-vault encrypted YAML data; and second, it only works on a single vault encrypted file at a
-time. To rekey everything in a large project you'd need to write a script that recursively
-goes through every file and rekeys every encrypted file and YAML variable all at once.
+suffers from two main drawbacks:
+
+1. It only works on vault encrypted files and not on vault encrypted YAML data
+2. It only works on a single vault encrypted file at a time.
+
+To rekey everything in a large project you'd need to write a script that goes through
+every file and rekeys everything in every format it can find.
 
 This is that script.
 
@@ -57,110 +58,49 @@ install `vault2vault` using [PipX](https://pypa.github.io/pipx/) and the `ansibl
 pipx install vault2vault[ansible]
 ```
 
-> Note: vault2vault requires an Ansible installation to function. If you are installing to
-> a standalone virtual environment (like with PipX) then you must install it with the
-> `ansible` extra to ensure a version of Ansible is available to the application.\*\*
+**Note: vault2vault requires an Ansible installation to function. If you are installing to a standalone virtual environment (like with PipX) then you must install it with the `ansible` extra to ensure a version of Ansible is available to the application.**
 
-## Usage
+## Using
 
-> Note: the full command reference is available by running `vault2vault --help`
+These docs are pretty sparse, largely because this project is still under active design
+and redevelopment. Here are the command line options:
 
-Vault2Vault works with files in any arbitrary directory structures, so there is no need to
-have your Ansible project(s) structured in a specific way for the tool to work. The
-simplest usage of Vault2Vault is by passing the path to your Ansible project directory to
-the command:
+```
+> vault2vault --help
+usage: vault2vault [-h] [--version] [--interactive] [-v] [-b] [-i VAULT_ID] [--ignore-undecryptable]
+                   [--old-pass-file OLD_PASS_FILE] [--new-pass-file NEW_PASS_FILE]
+                   [paths ...]
 
-```bash
-vault2vault ./my-ansible-project/
+Recursively rekey ansible-vault encrypted files and in-line variables
+
+positional arguments:
+  paths                 Paths to search for Ansible Vault encrypted content
+
+options:
+  -h, --help            show this help message and exit
+  --version             Show program version and exit
+  --interactive         Step through files and variables interactively, prompting for confirmation before making
+                        each change
+  -v, --verbose         Increase verbosity; can be repeated
+  -b, --backup          Write a backup of every file to be modified, suffixed with '.bak'
+  -i VAULT_ID, --vault-id VAULT_ID
+                        Limit rekeying to encrypted secrets with the specified Vault ID
+  --ignore-undecryptable
+                        Ignore any file or variable that is not decryptable with the provided vault secret instead
+                        of raising an error
+  --old-pass-file OLD_PASS_FILE
+                        Path to a file with the old vault password to decrypt secrets with
+  --new-pass-file NEW_PASS_FILE
+                        Path to a file with the new vault password to rekey secrets with
 ```
 
-The tool will prompt for the current vault password and the new vault password and then
-process every file under the provided path. You can also specify multiple paths and
-they'll all be processed together:
-
-```bash
-vault2vault \
-  ./my-ansible-project/playbooks/ \
-  ./my-ansible-project/host_vars/ \
-  ./my-ansible-project/group_vars/
-```
-
-To skip the interactive password prompts you can put the password in a file and have the
-tool read it in at runtime. The `--old-pass-file` and `--new-pass-file` parameters work
-the same way as the `--vault-password-file` option from the `ansible` command:
-
-```bash
-vault2vault ./my-ansible-project/ \
-  --old-pass-file=./oldpass.txt \
-  --new-pass-file=./newpass.txt
-```
-
-If you use multiple vault passwords in your project and want to roll them you'll need to
-run `vault2vault` once for each password you want to change. By default, `vault2vault`
-will fail with an error if it encounters vaulted data that it cannot decrypt with the
-provided current vault password. To change this behavior and instead just ignore any
-vaulted data that can't be decrypted (like, for example, if you have data encrypted with
-multiple vault passwords) you can pass the `--ignore-undecryptable` flag to turn the
-errors into warnings.
-
-> Please report any bugs or issues you encounter on
-> [Github](https://github.com/enpaul/vault2vault/issues).
-
-### Recovering from a failed migration
-
-This tool is still pretty early in it's development, and to be honest it hooks into
-Ansible's functionality in some fragile ways. I've tested as best I can to ensure it
-covers as many edge cases as possible, but there is still the chance that you might get
-partway through a password migration and then have the tool fail out, leaving half of your
-data successfully rekeyed and the other half not.
-
-In the spirit of the
-[Unix philosophy](https://hackaday.com/2018/09/10/doing-one-thing-well-the-unix-philosophy/)
-this tool does not include any built-in way to recover from this state. However, it can be
-done very effectively using a version control tool.
-
-If you are using Git to track your project files then you can use the command
-`git reset --hard` to restore all files to the state of the currently checked out commit.
-This does have the side effect of erasing any other un-committed work in the repository,
-so it's recommended to always have a clean working tree when using Vault2Vault.
-
-If you are not using a version control system to track your project files then you can
-create a temporary Git repository to use in the event of a migration failure:
-
-```bash
-cd my-project/
-
-# Initialize the new repository
-git init
-
-# Add and commit all your existing files to the git tree
-git add .
-git commit -m "initial commit"
-
-# Run vault migrations
-vault2vault ...
-
-# If no recovery is necessary, delete the git repository data
-rm -rf .git
-```
-
-## Roadmap
-
-This project is considered feature complete as of the
-[0.1.1](https://github.com/enpaul/vault2vault/releases/tag/0.1.1) release. As a result the
-roadmap focuses on stability and user experience ahead of a 1.0 release.
-
-- [ ] Reimplement core vaulted data processing function to enable multithreading
-- [ ] Implement multithreading for performance in large environments
-- [ ] Add unit tests
-- [ ] Add integration tests
-- [ ] Redesign logging messages to improve clarity and consistency
+Please report any bugs or issues you encounter on
+[Github](https://github.com/enpaul/vault2vault/issues).
 
 ## Developer Documentation
 
 All project contributors and participants are expected to adhere to the
-[Contributor Covenant Code of Conduct, v2](CODE_OF_CONDUCT.md)
-([external link](https://www.contributor-covenant.org/version/2/0/code_of_conduct/)).
+[Contributor Covenant Code of Conduct, v2](CODE_OF_CONDUCT.md) ([external link](https://www.contributor-covenant.org/version/2/0/code_of_conduct/)).
 
 The `devel` branch has the latest (and potentially unstable) changes. The stable releases
 are tracked on [Github](https://github.com/enpaul/vault2vault/releases),

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vault2vault"
-version = "0.1.3"
+version = "0.1.1"
 license = "MIT"
 authors = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
 description = "Recursively rekey ansible-vault encrypted files and in-line variables"
@@ -12,7 +12,7 @@ packages = [
 keywords = ["ansible", "vault", "playbook", "yaml", "password"]
 readme = "README.md"
 classifiers = [
-  "Development Status :: 4 - Beta",
+  "Development Status :: 3 - Alpha",
   "Environment :: Console",
   "Framework :: Ansible",
   "Intended Audience :: Developers",
@@ -22,11 +22,11 @@ classifiers = [
   "Natural Language :: English",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.6",
   "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: Implementation :: CPython"
 ]
 
@@ -37,43 +37,30 @@ vault2vault = "vault2vault:main"
 ansible = ["ansible-core"]
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.6.1"
 "ruamel.yaml" = "^0.17.16"
 ansible-core = {version = "^2.11.5", optional = true}
 
-[tool.poetry.group.dev.dependencies]
-black = {version = "^23.1.0", python = "^3.10"}
-blacken-docs = {version = "^1.13.0", python = "^3.10"}
-ipython = {version = "^8.10.1", python = "^3.10"}
-mdformat = {version = "^0.7.16", python = "^3.10"}
-mdformat-gfm = {version = "^0.3.5", python = "^3.10"}
-mypy = {version = "^1.1.1", python = "^3.10"}
-pre-commit = {version = "^2.7.1", python = "^3.10"}
-pre-commit-hooks = {version = "^3.3.0", python = "^3.10"}
-pylint = {version = "^2.4.4", python = "^3.10"}
-reorder-python-imports = {version = "^2.3.5", python = "^3.10"}
-types-toml = {version = "^0.10.4", python = "^3.10"}
-# Implicit python version check fails for this one
-packaging = {version = "^23.0", python = "^3.10"}
-
-[tool.poetry.group.security.dependencies]
-bandit = {version = "^1.6.2", python = "^3.10"}
-safety = {version = "^2.2.0", python = "^3.10"}
-poetry = {version = "^1.2.0", python = "^3.10"}
-
-[tool.poetry.group.test.dependencies]
-pytest = {version = "^6.0.2"}
-pytest-cov = {version = "^2.10.1"}
-toml = {version = "^0.10.1"}
-typing-extensions = {version = "^4.5.0", python = "^3.8"}
-
-[tool.poetry.group.ci.dependencies]
-tox = {version = "^3.20.0"}
-tox-poetry-installer = {version = "^0.10.1", extras = ["poetry"]}
-# This doesn't get installed under py3.7 for some reason, but it's
-# required for poetry. Will need to debug this more in the future
-backports-cached-property = "^1.0.2"
+[tool.poetry.dev-dependencies]
+bandit = "^1.6.2"
+black = { version = "^21.9b0", allow-prereleases = true, python = "^3.7" }
+blacken-docs = "^1.8.0"
+ipython = { version = "^7.18.1", python = "^3.7" }
+mypy = "^0.800"
+pre-commit = "^2.7.1"
+pre-commit-hooks = "^3.3.0"
+pylint = "^2.4.4"
+pytest = "^6.0.2"
+pytest-cov = "^2.10.1"
+reorder-python-imports = "^2.3.5"
+safety = "^1.9.0"
+toml = "^0.10.1"
+tox = "^3.20.0"
+tox-poetry-installer = { version = "^0.8.1", extras = ["poetry"] }
+types-toml = "^0.10.4"
+mdformat = "^0.6.4"
+mdformat-gfm = "^0.2"
 
 [build-system]
-requires = ["poetry-core>=1.1.0"]
+requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py3{7-11}, static, static-tests, security
+envlist = py36, py37, py38, py39, py310, static, static-tests, security
 isolated_build = true
 skip_missing_interpreters = true
 
@@ -9,8 +9,10 @@ require_locked_deps = true
 require_poetry = true
 extras =
     ansible
-poetry_dep_groups =
-    test
+locked_deps =
+    pytest
+    pytest-cov
+    toml
 commands =
     pytest {toxinidir}/tests/ \
       --cov vault2vault \
@@ -19,11 +21,20 @@ commands =
 
 [testenv:static]
 description = Static formatting and quality enforcement
-basepython = python3.10
+basepython = python3.8
 platform = linux
 ignore_errors = true
-poetry_dep_groups =
-    dev
+locked_deps =
+    black
+    blacken-docs
+    mdformat
+    mdformat-gfm
+    mypy
+    reorder-python-imports
+    pre-commit
+    pre-commit-hooks
+    pylint
+    types-toml
 commands =
     pre-commit run \
       --all-files
@@ -35,7 +46,7 @@ commands =
 
 [testenv:static-tests]
 description = Static formatting and quality enforcement for the tests
-basepython = python3.10
+basepython = python3.8
 platform = linux
 ignore_errors = true
 locked_deps =
@@ -52,12 +63,14 @@ commands =
 
 [testenv:security]
 description = Security checks
-basepython = python3.10
+basepython = python3.8
 platform = linux
 ignore_errors = true
 skip_install = true
-poetry_dep_groups =
-    security
+locked_deps =
+    bandit
+    safety
+    poetry
 commands =
     bandit {toxinidir}/vault2vault.py \
       --recursive \
@@ -69,14 +82,8 @@ commands =
     poetry export \
       --format requirements.txt \
       --output {envtmpdir}/requirements.txt \
-      --without-hashes
-# For now these groups are disabled until this bug is resolved
-# in poetry-plugin-export:
-# https://github.com/python-poetry/poetry-plugin-export/issues/176
-#      --with dev \
-#      --with ci \
-#      --with security \
-#      --with test
+      --without-hashes \
+      --dev
     safety check \
       --file {envtmpdir}/requirements.txt \
       --json

vault2vault.py

@@ -28,7 +28,7 @@ except ImportError:
 
 __title__ = "vault2vault"
 __summary__ = "Recursively rekey ansible-vault encrypted files and in-line variables"
-__version__ = "0.1.3"
+__version__ = "0.1.1"
 __url__ = "https://github.com/enpaul/vault2vault/"
 __license__ = "MIT"
 __authors__ = ["Ethan Paul <24588726+enpaul@users.noreply.github.com>"]
@@ -73,19 +73,6 @@ def _process_file(  # pylint: disable=too-many-statements
     backup: bool,
     ignore: bool,
 ) -> None:
-    """Determine whether a filepath includes vaulted data and if so, rekey it
-
-    :param path: Path to the file to check
-    :param old: VaultLib object with the current (old) vault password encoded in it
-    :param new: VaultLib object with the target (new) vault password encoded in it
-    :param interactive: Whether to prompt interactively for confirmation before each
-                        rekey operation
-    :param backup: Whether to copy the original file to a backup before making any
-                   in-place changes
-    :param ignore: Whether to ignore any errors that come from failing to decrypt
-                   any vaulted data
-    """
-
     logger = logging.getLogger(__name__)
 
     logger.debug(f"Processing file {path}")
@@ -361,7 +348,6 @@ def _load_password(
                   the password will be prompted for interactively.
     :param desc: Description text to inject into the interactive password prompt. Useful when using
                  this function multiple times to identify different passwords to the user.
-    :param confirm: Whether to prompt twice for the input and check that the two inputs match
     :returns: Populated vault secret object with the loaded password
     """
 
@@ -369,7 +355,7 @@ def _load_password(
 
     if fpath:
         try:
-            with Path(fpath).resolve().open("rb") as infile:
+            with Path(fpath).resolve().open("rb", encoding="utf-8") as infile:
                 return VaultSecret(infile.read())
         except (FileNotFoundError, PermissionError) as err:
             raise RuntimeError(