diff --git a/.github/actions/security-issues/action.yml b/.github/actions/security-issues/action.yml index 6fca189c3..1a857aa9c 100644 --- a/.github/actions/security-issues/action.yml +++ b/.github/actions/security-issues/action.yml @@ -39,7 +39,7 @@ runs: - name: Install Python Toolbox / Security tool shell: bash run: | - pip install exasol-toolbox==6.1.0 + pip install exasol-toolbox==6.1.1 - name: Create Security Issue Report shell: bash diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index e49fd1aab..7b0d6eb9c 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,7 @@ # Changelog * [unreleased](unreleased.md) +* [6.1.1](changes_6.1.1.md) * [6.1.0](changes_6.1.0.md) * [6.0.0](changes_6.0.0.md) * [5.1.1](changes_5.1.1.md) @@ -59,6 +60,7 @@ hidden: --- unreleased +changes_6.1.1 changes_6.1.0 changes_6.0.0 changes_5.1.1 diff --git a/doc/changes/changes_6.1.1.md b/doc/changes/changes_6.1.1.md new file mode 100644 index 000000000..53e28d3a9 --- /dev/null +++ b/doc/changes/changes_6.1.1.md @@ -0,0 +1,32 @@ +# 6.1.1 - 2026-03-18 + +## Summary + +## Security Issues + +* #748: Updated dependency to `black` + +## Refactorings + +* #752: Updated upload-artifact from v6 to v7 and download-artifact from v7 to v8 +* #750: Updated dependency `pip-audit` + +## Dependency Updates + +### `main` + +* Updated dependency `bandit:1.9.3` to `1.9.4` +* Updated dependency `black:25.12.0` to `26.3.1` +* Updated dependency `coverage:7.13.1` to `7.13.4` +* Updated dependency `import-linter:2.9` to `2.11` +* Updated dependency `nox:2025.11.12` to `2026.2.9` +* Updated dependency `pip-audit:2.9.0` to `2.10.0` +* Updated dependency `pip-licenses:5.5.0` to `5.5.1` +* Updated dependency `pylint:4.0.4` to `4.0.5` +* Updated dependency `ruff:0.14.13` to `0.14.14` +* Updated dependency `sphinxcontrib-mermaid:2.0.0` to `2.0.1` +* Updated dependency `typer:0.21.1` to `0.24.1` + +### `dev` + +* Updated dependency `cookiecutter:2.6.0` to `2.7.1` diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index a4f3ffcf8..fb4737052 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,11 +1,3 @@ # Unreleased ## Summary - -## Security Issues - -* #748: Updated dependency to `black` - -## Refactoring - -* #752: Updated upload-artifact from v6 to v7 and download-artifact from v7 to v8 diff --git a/exasol/toolbox/nox/_dependencies.py b/exasol/toolbox/nox/_dependencies.py index 45f28a3ca..dc860875a 100644 --- a/exasol/toolbox/nox/_dependencies.py +++ b/exasol/toolbox/nox/_dependencies.py @@ -35,7 +35,7 @@ def audit(session: Session) -> None: try: vulnerabilities = Vulnerabilities.load_from_pip_audit(working_directory=Path()) except PipAuditException as e: - session.error(e.return_code, e.stdout, e.stderr) + session.error(e.returncode, e.stdout, e.stderr) security_issue_dict = vulnerabilities.security_issue_dict print(json.dumps(security_issue_dict, indent=2)) diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 1e6281df3..53fb67352 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -27,16 +27,18 @@ ) +PipAuditEntry = dict[str, str | list[str] | tuple[str, ...]] + + @dataclass class PipAuditException(Exception): - return_code: int + returncode: int stdout: str stderr: str - def __init__(self, subprocess_output: subprocess.CompletedProcess) -> None: - self.return_code = subprocess_output.returncode - self.stdout = subprocess_output.stdout - self.stderr = subprocess_output.stderr + @classmethod + def from_subprocess(cls, proc: subprocess.CompletedProcess) -> PipAuditException: + return cls(proc.returncode, proc.stdout, proc.stderr) class VulnerabilitySource(str, Enum): @@ -102,7 +104,7 @@ def reference_links(self) -> tuple[str, ...]: ) @property - def security_issue_entry(self) -> dict[str, str | list[str] | tuple[str, ...]]: + def security_issue_entry(self) -> PipAuditEntry: return { "name": self.package.name, "version": str(self.package.version), @@ -132,10 +134,20 @@ def subsection_for_changelog_summary(self) -> str: """ Create a subsection to be included in the Summary section of a versioned changelog. """ - links_join = "\n* ".join(sorted(self.reference_links)) - references_subsection = f"\n#### References:\n\n* {links_join}\n\n " - subsection = f"### {self.vulnerability_id} in {self.package.coordinates}\n\n{self.description}\n{references_subsection}" - return cleandoc(subsection.strip()) + indent = " " * 12 + references = f"\n{indent}".join( + f"* {link}" for link in sorted(self.reference_links) + ) + description = self.description.replace("\n", f"\n{indent}") + return cleandoc(f""" + ### {self.vulnerability_id} in {self.package.coordinates} + + {description} + + #### References + + {references} + """) def audit_poetry_files(working_directory: Path) -> str: @@ -159,7 +171,7 @@ def audit_poetry_files(working_directory: Path) -> str: cwd=working_directory, ) # nosec if output.returncode != 0: - raise PipAuditException(subprocess_output=output) + raise PipAuditException.from_subprocess(output) with tempfile.TemporaryDirectory() as path: tmpdir = Path(path) @@ -179,7 +191,7 @@ def audit_poetry_files(working_directory: Path) -> str: # they both map to returncode = 1, so we have our own logic to raise errors # for the case of 2) and not 1). if not search(PIP_AUDIT_VULNERABILITY_PATTERN, output.stderr.strip()): - raise PipAuditException(subprocess_output=output) + raise PipAuditException.from_subprocess(output) return output.stdout @@ -215,7 +227,7 @@ def load_from_pip_audit(cls, working_directory: Path) -> Vulnerabilities: return Vulnerabilities(vulnerabilities=vulnerabilities) @property - def security_issue_dict(self) -> list[dict[str, str | list[str] | tuple[str, ...]]]: + def security_issue_dict(self) -> list[PipAuditEntry]: return [ vulnerability.security_issue_entry for vulnerability in self.vulnerabilities ] diff --git a/exasol/toolbox/version.py b/exasol/toolbox/version.py index d43c49466..46c89c6a8 100644 --- a/exasol/toolbox/version.py +++ b/exasol/toolbox/version.py @@ -10,6 +10,6 @@ MAJOR = 6 MINOR = 1 -PATCH = 0 +PATCH = 1 VERSION = f"{MAJOR}.{MINOR}.{PATCH}" __version__ = VERSION diff --git a/poetry.lock b/poetry.lock index 2747410af..b0081efa0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2246,32 +2246,34 @@ pip = "*" [[package]] name = "pip-audit" -version = "2.9.0" +version = "2.10.0" description = "A tool for scanning Python environments for known vulnerabilities" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pip_audit-2.9.0-py3-none-any.whl", hash = "sha256:348b16e60895749a0839875d7cc27ebd692e1584ebe5d5cb145941c8e25a80bd"}, - {file = "pip_audit-2.9.0.tar.gz", hash = "sha256:0b998410b58339d7a231e5aa004326a294e4c7c6295289cdc9d5e1ef07b1f44d"}, + {file = "pip_audit-2.10.0-py3-none-any.whl", hash = "sha256:16e02093872fac97580303f0848fa3ad64f7ecf600736ea7835a2b24de49613f"}, + {file = "pip_audit-2.10.0.tar.gz", hash = "sha256:427ea5bf61d1d06b98b1ae29b7feacc00288a2eced52c9c58ceed5253ef6c2a4"}, ] [package.dependencies] CacheControl = {version = ">=0.13.0", extras = ["filecache"]} -cyclonedx-python-lib = ">=5,<10" +cyclonedx-python-lib = ">=5,<12" packaging = ">=23.0.0" pip-api = ">=0.0.28" pip-requirements-parser = ">=32.0.0" platformdirs = ">=4.2.0" requests = ">=2.31.0" rich = ">=12.4" -toml = ">=0.10" +tomli = ">=2.2.1" +tomli-w = ">=1.2.0" [package.extras] +cov = ["coverage[toml] (>=7.0,!=7.3.3,<8.0)"] dev = ["build", "pip-audit[doc,lint,test]"] doc = ["pdoc"] -lint = ["interrogate (>=1.6,<2.0)", "mypy", "ruff (>=0.9,<1.0)", "types-requests", "types-toml"] -test = ["coverage[toml] (>=7.0,!=7.3.3,<8.0)", "pretend", "pytest", "pytest-cov"] +lint = ["interrogate (>=1.6,<2.0)", "mypy", "ruff (>=0.11)", "types-requests", "types-toml"] +test = ["pip-audit[cov]", "pretend", "pytest"] [[package]] name = "pip-licenses" @@ -3771,7 +3773,7 @@ version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -groups = ["main", "dev"] +groups = ["dev"] files = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, @@ -3820,6 +3822,18 @@ files = [ ] markers = {dev = "python_version == \"3.10\""} +[[package]] +name = "tomli-w" +version = "1.2.0" +description = "A lil' TOML writer" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90"}, + {file = "tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021"}, +] + [[package]] name = "tomlkit" version = "0.14.0" @@ -4024,4 +4038,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "99304f1ee69b51728cd4e92b07f54a5b8735be672f79d77311481c6d8b837e76" +content-hash = "3d5c07aeaab839a92ec06e66addd20d634864518ef66d76623d08d5eaae6817b" diff --git a/project-template/cookiecutter.json b/project-template/cookiecutter.json index c9cd8fe58..c30b80a3f 100644 --- a/project-template/cookiecutter.json +++ b/project-template/cookiecutter.json @@ -9,7 +9,7 @@ "author_email": "opensource@exasol.com", "project_short_tag": "", "python_version_min": "3.10", - "exasol_toolbox_version_range": ">=6.1.0,<7", + "exasol_toolbox_version_range": ">=6.1.1,<7", "license_year": "{% now 'utc', '%Y' %}", "__repo_name_slug": "{{cookiecutter.package_name}}", "__package_name_slug": "{{cookiecutter.package_name}}", diff --git a/pyproject.toml b/pyproject.toml index b66ec2fee..3fe0f1cae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "exasol-toolbox" -version = "6.1.0" +version = "6.1.1" description = "Your one-stop solution for managing all standard tasks and core workflows of your Python project." authors = [ { name = "Nicola Coretti", email = "nicola.coretti@exasol.com" }, @@ -38,7 +38,7 @@ dependencies = [ "mypy>=0.971", "myst-parser>=2.0.0,<4", "nox>=2022.8.7", - "pip-audit>=2.7.3,<2.10.0", # see issue https://github.com/exasol/python-toolbox/issues/750 + "pip-audit>=2.10,<3", "pip-licenses>=5.0.0,<6", "pluggy>=1.5.0,<2", "pre-commit>=4,<5", diff --git a/test/conftest.py b/test/conftest.py index ab5fe8e5e..8c7b69221 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,7 +8,10 @@ Issue, _issues_as_json_str, ) -from exasol.toolbox.util.dependencies.audit import Vulnerability +from exasol.toolbox.util.dependencies.audit import ( + PipAuditEntry, + Vulnerability, +) class SampleVulnerability: @@ -24,11 +27,11 @@ class SampleVulnerability: ) @property - def pip_audit_vuln_entry(self) -> dict[str, str | list[str]]: + def pip_audit_vuln_entry(self) -> PipAuditEntry: return { - "id": self.vulnerability_id, + "id": self.cve_id, "fix_versions": [self.fix_version], - "aliases": [self.cve_id], + "aliases": [self.vulnerability_id], "description": self.description, } @@ -59,7 +62,7 @@ def nox_dependencies_audit(self) -> str: return json.dumps([self.security_issue_entry], indent=2) + "\n" @property - def security_issue_entry(self) -> dict[str, str | list[str] | tuple[str, ...]]: + def security_issue_entry(self) -> PipAuditEntry: return { "name": self.package_name, "version": self.version, diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 2f4c9814f..4fb7039e1 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -1,10 +1,26 @@ import subprocess +from pathlib import Path import pytest +from exasol.toolbox.config import BaseConfig + @pytest.fixture(scope="session") def poetry_path() -> str: result = subprocess.run(["which", "poetry"], capture_output=True, text=True) poetry_path = result.stdout.strip() return poetry_path + + +@pytest.fixture(scope="session") +def ptb_minimum_python_version() -> str: + """ + Some integration tests create a sample poetry project and need to + specify its minimum python version in property "requires-python" in file + pyproject.toml. + + This fixture returns a value including all python versions supported by + the PTB. + """ + return BaseConfig(root_path=Path(), project_name="toolbox").minimum_python_version diff --git a/test/integration/smoke_test.py b/test/integration/smoke_test.py deleted file mode 100644 index f6284a457..000000000 --- a/test/integration/smoke_test.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_passing_integration_test(): - assert True diff --git a/test/integration/util/dependencies/audit_integration_test.py b/test/integration/util/dependencies/audit_integration_test.py index 914a402e5..96da23ff0 100644 --- a/test/integration/util/dependencies/audit_integration_test.py +++ b/test/integration/util/dependencies/audit_integration_test.py @@ -1,62 +1,126 @@ +from __future__ import annotations + import json +import re import subprocess from inspect import cleandoc +from pathlib import Path import pytest -from exasol.toolbox.util.dependencies.audit import audit_poetry_files +from exasol.toolbox.util.dependencies.audit import ( + PipAuditEntry, + audit_poetry_files, +) + + +def aux_subprocess(*cmd, **kwargs) -> subprocess.CompletedProcess: + """ + Runs the specified command via subprocess with default kwargs suitable + for running auxiliary commands, e.g. in pytest fixtures. + + The command is an executable incl its args and CLI options. + + The default kwargs are + + * Raise an exception on non-zero returncode + * Capture output to keep pytest output clean + * Use empty environment (no env variables) + """ + kwargs_with_defaults = { + "env": {}, + "check": True, + "capture_output": True, + } | kwargs + return subprocess.run(cmd, **kwargs_with_defaults) + + +class PoetryProject: + def __init__(self, poetry_path: Path, path: Path): + self.poetry = poetry_path + self.dir = path + + @property + def name(self) -> str: + return self.dir.name + + @property + def toml(self) -> Path: + return self.dir / "pyproject.toml" + + def create(self) -> PoetryProject: + aux_subprocess(self.poetry, "new", self.name, cwd=self.dir.parent) + return self + + def set_minimum_python_version(self, version: str) -> PoetryProject: + content = self.toml.read_text() + changed = re.sub( + r'^requires-python = ".*"$', + f'requires-python = ">={version}"', + content, + flags=re.MULTILINE, + ) + self.toml.write_text(changed) + return self + + def add_package(self, spec: str) -> PoetryProject: + aux_subprocess(self.poetry, "add", spec, cwd=self.dir) + return self + + def add_to_toml(self, content: str) -> PoetryProject: + with self.toml.open("a") as f: + f.write(cleandoc(content)) + return self + + def install(self) -> PoetryProject: + aux_subprocess(self.poetry, "install", cwd=self.dir) + return self @pytest.fixture -def create_poetry_project(tmp_path, sample_vulnerability, poetry_path): - project_name = "vulnerability" - subprocess.run([poetry_path, "new", project_name], cwd=tmp_path, env={}) - - poetry_root_dir = tmp_path / project_name - subprocess.run( - [ - poetry_path, - "add", - f"{sample_vulnerability.package_name}=={sample_vulnerability.version}", - ], - cwd=poetry_root_dir, - env={}, +def create_poetry_project( + tmp_path, sample_vulnerability, poetry_path, ptb_minimum_python_version +): + project = ( + PoetryProject(poetry_path, tmp_path / "vulnerability") + .create() + .set_minimum_python_version(ptb_minimum_python_version) + .add_package( + f"{sample_vulnerability.package_name}==" f"{sample_vulnerability.version}" + ) + .add_to_toml(""" + [tool.poetry.requires-plugins] + poetry-plugin-export = ">=1.8" + """) + .install() ) + return project.dir - poetry_export = cleandoc(""" - [tool.poetry.requires-plugins] - poetry-plugin-export = ">=1.8" - """) - with (poetry_root_dir / "pyproject.toml").open("a") as f: - f.write(poetry_export) +def without_vuln_descriptions(dep: PipAuditEntry): + def strip_description(entry: PipAuditEntry): + return {k: v for k, v in entry.items() if k != "description"} - subprocess.run( - [poetry_path, "install"], - cwd=poetry_root_dir, - env={}, - ) + def without_descriptions(vulnerabilities): + return [strip_description(v) for v in vulnerabilities] - return poetry_root_dir + return {k: (without_descriptions(v) if k == "vulns" else v) for k, v in dep.items()} -class TestAuditPoetryFiles: - @staticmethod - def test_works_as_expected(create_poetry_project, sample_vulnerability): - result = audit_poetry_files(working_directory=create_poetry_project) - expected_innards = sample_vulnerability.pip_audit_vuln_entry.copy() - expected_innards.pop("description") +def find_dependency(dependencies: list[PipAuditEntry], name: str) -> PipAuditEntry: + generator = (d for d in dependencies if d["name"] == name) + return next(generator) - assert isinstance(result, str) - result_dict = json.loads(result) - for entry in result_dict["dependencies"]: - if entry["name"] == sample_vulnerability.package_name: - for vuln in entry["vulns"]: - vuln.pop("description") +def test_pip_audit(create_poetry_project, sample_vulnerability): + vuln = sample_vulnerability + audit_output = audit_poetry_files(working_directory=create_poetry_project) + result = json.loads(audit_output) + actual = find_dependency(result["dependencies"], vuln.package_name) - assert entry == { - "name": sample_vulnerability.package_name, - "version": sample_vulnerability.version, - "vulns": [expected_innards], - } + expected = { + "name": vuln.package_name, + "version": vuln.version, + "vulns": [vuln.pip_audit_vuln_entry], + } + assert without_vuln_descriptions(actual) == without_vuln_descriptions(expected) diff --git a/test/unit/util/dependencies/audit_test.py b/test/unit/util/dependencies/audit_test.py index 2ae81b6c2..ed9ce8403 100644 --- a/test/unit/util/dependencies/audit_test.py +++ b/test/unit/util/dependencies/audit_test.py @@ -34,8 +34,8 @@ def test_from_audit_entry(sample_vulnerability): result = sample_vulnerability.vulnerability assert result == Vulnerability( package=sample_vulnerability.vulnerability.package, - id=sample_vulnerability.vulnerability_id, - aliases=[sample_vulnerability.cve_id], + id=sample_vulnerability.cve_id, + aliases=[sample_vulnerability.vulnerability_id], fix_versions=[sample_vulnerability.fix_version], description=sample_vulnerability.description, ) @@ -113,7 +113,7 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability): `|attr` filter allows an attacker that controls the content of a template to execute arbitrary Python code. - #### References: + #### References * https://github.com/advisories/GHSA-cpwx-vrp4-4pq7 * https://nvd.nist.gov/vuln/detail/CVE-2025-27516