From 300c5558ab051a978e0abe04df963ef302eb0958 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 30 Oct 2025 22:33:29 +0100 Subject: [PATCH] Update `check_dependencies` to support markers (#19110) --- changelog.d/19110.misc | 1 + synapse/util/check_dependencies.py | 131 ++++++++++++++++++++++++-- tests/util/test_check_dependencies.py | 52 +++++++++- 3 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 changelog.d/19110.misc diff --git a/changelog.d/19110.misc b/changelog.d/19110.misc new file mode 100644 index 000000000..dc45eef17 --- /dev/null +++ b/changelog.d/19110.misc @@ -0,0 +1 @@ +Allow Synapse's runtime dependency checking code to take packaging markers (i.e. `python <= 3.14`) into account when checking dependencies. \ No newline at end of file diff --git a/synapse/util/check_dependencies.py b/synapse/util/check_dependencies.py index 1c79c0be4..715240c8c 100644 --- a/synapse/util/check_dependencies.py +++ b/synapse/util/check_dependencies.py @@ -28,8 +28,9 @@ require. But this is probably just symptomatic of Python's package management. import logging from importlib import metadata -from typing import Iterable, NamedTuple, Optional +from typing import Any, Iterable, NamedTuple, Optional, Sequence, cast +from packaging.markers import Marker, Value, Variable, default_environment from packaging.requirements import Requirement DISTRIBUTION_NAME = "matrix-synapse" @@ -65,9 +66,23 @@ RUNTIME_EXTRAS = set(ALL_EXTRAS) - DEV_EXTRAS VERSION = metadata.version(DISTRIBUTION_NAME) +def _marker_environment(extra: str) -> dict[str, str]: + """Return the marker environment for `extra`, seeded with the current interpreter.""" + + env = cast(dict[str, str], dict(default_environment())) + env["extra"] = extra + return env + + def _is_dev_dependency(req: Requirement) -> bool: - return req.marker is not None and any( - req.marker.evaluate({"extra": e}) for e in DEV_EXTRAS + """Return True if `req` is a development dependency.""" + if req.marker is None: + return False + + marker_extras = _extras_from_marker(req.marker) + return any( + extra in DEV_EXTRAS and req.marker.evaluate(_marker_environment(extra)) + for extra in marker_extras ) @@ -95,6 +110,7 @@ def _generic_dependencies() -> Iterable[Dependency]: """Yield pairs (requirement, must_be_installed).""" requirements = metadata.requires(DISTRIBUTION_NAME) assert requirements is not None + env_no_extra = _marker_environment("") for raw_requirement in requirements: req = Requirement(raw_requirement) if _is_dev_dependency(req) or _should_ignore_runtime_requirement(req): @@ -103,7 +119,7 @@ def _generic_dependencies() -> Iterable[Dependency]: # https://packaging.pypa.io/en/latest/markers.html#usage notes that # > Evaluating an extra marker with no environment is an error # so we pass in a dummy empty extra value here. - must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""}) + must_be_installed = req.marker is None or req.marker.evaluate(env_no_extra) yield Dependency(req, must_be_installed) @@ -111,6 +127,8 @@ def _dependencies_for_extra(extra: str) -> Iterable[Dependency]: """Yield additional dependencies needed for a given `extra`.""" requirements = metadata.requires(DISTRIBUTION_NAME) assert requirements is not None + env_no_extra = _marker_environment("") + env_for_extra = _marker_environment(extra) for raw_requirement in requirements: req = Requirement(raw_requirement) if _is_dev_dependency(req): @@ -118,12 +136,84 @@ def _dependencies_for_extra(extra: str) -> Iterable[Dependency]: # Exclude mandatory deps by only selecting deps needed with this extra. if ( req.marker is not None - and req.marker.evaluate({"extra": extra}) - and not req.marker.evaluate({"extra": ""}) + and req.marker.evaluate(env_for_extra) + and not req.marker.evaluate(env_no_extra) ): yield Dependency(req, True) +def _values_from_marker_value(value: Value) -> set[str]: + """Extract text values contained in a marker `Value`.""" + + raw: Any = value.value + if isinstance(raw, str): + return {raw} + if isinstance(raw, (tuple, list)): + return {str(item) for item in raw} + return {str(raw)} + + +def _extras_from_marker(marker: Optional[Marker]) -> set[str]: + """Return every `extra` referenced in the supplied marker tree.""" + + extras: set[str] = set() + + if marker is None: + return extras + + def collect(tree: object) -> None: + if isinstance(tree, list): + for item in tree: + collect(item) + elif isinstance(tree, tuple) and len(tree) == 3: + lhs, _op, rhs = tree + if ( + isinstance(lhs, Variable) + and lhs.value == "extra" + and isinstance(rhs, Value) + ): + extras.update(_values_from_marker_value(rhs)) + elif ( + isinstance(rhs, Variable) + and rhs.value == "extra" + and isinstance(lhs, Value) + ): + extras.update(_values_from_marker_value(lhs)) + + collect(marker._markers) + return extras + + +def _extras_to_consider_for_requirement( + marker: Marker, base_candidates: Sequence[str] +) -> set[str]: + """ + Augment `base_candidates` with extras explicitly mentioned in `marker`. + + Markers can mention extras (e.g. `extra == "saml2"`). + """ + + # Avoid modifying the input sequence. + # Use a set to efficiently avoid duplicate extras. + extras = set(base_candidates) + + for candidate in _extras_from_marker(marker): + extras.add(candidate) + + return extras + + +def _marker_applies_for_any_extra(requirement: Requirement, extras: set[str]) -> bool: + """Check whether a requirement's marker matches any evaluated `extra`.""" + + if requirement.marker is None: + return True + + return any( + requirement.marker.evaluate(_marker_environment(extra)) for extra in extras + ) + + def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str: if extra: return ( @@ -164,7 +254,7 @@ def _no_reported_version(requirement: Requirement, extra: Optional[str] = None) def check_requirements(extra: Optional[str] = None) -> None: """Check Synapse's dependencies are present and correctly versioned. - If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in + If provided, `extra` must be the name of an packaging extra (e.g. "saml2" in `pip install matrix-synapse[saml2]`). If `extra` is None, this function checks that @@ -174,6 +264,15 @@ def check_requirements(extra: Optional[str] = None) -> None: If `extra` is not None, this function checks that - the dependencies needed for that extra are installed and correctly versioned. + `marker`s are optional attributes on each requirement which specify + conditions under which the requirement applies. For example, a requirement + might only be needed on Windows, or with Python < 3.14. Markers can + additionally mention `extras` themselves, meaning a requirement may not + apply if the marker mentions an extra that the user has not asked for. + + This function skips a requirement when its markers do not apply in the + current environment. + :raises DependencyException: if a dependency is missing or incorrectly versioned. :raises ValueError: if this extra does not exist. """ @@ -188,7 +287,25 @@ def check_requirements(extra: Optional[str] = None) -> None: deps_unfulfilled = [] errors = [] + if extra is None: + # Default to all mandatory dependencies (non-dev extras). + # "" means all dependencies that aren't conditional on an extra. + base_extra_candidates: Sequence[str] = ("", *RUNTIME_EXTRAS) + else: + base_extra_candidates = (extra,) + for requirement, must_be_installed in dependencies: + if requirement.marker is not None: + candidate_extras = _extras_to_consider_for_requirement( + requirement.marker, base_extra_candidates + ) + # Skip checking this dependency if the requirement's marker object + # (i.e. `python_version < "3.14" and os_name == "win32"`) does not + # apply for any of the extras we're considering. + if not _marker_applies_for_any_extra(requirement, candidate_extras): + continue + + # Check if the requirement is installed and correctly versioned. try: dist: metadata.Distribution = metadata.distribution(requirement.name) except metadata.PackageNotFoundError: diff --git a/tests/util/test_check_dependencies.py b/tests/util/test_check_dependencies.py index c052ba2b7..ab2e2f629 100644 --- a/tests/util/test_check_dependencies.py +++ b/tests/util/test_check_dependencies.py @@ -22,9 +22,11 @@ from contextlib import contextmanager from os import PathLike from pathlib import Path -from typing import Generator, Optional, Union +from typing import Generator, Optional, Union, cast from unittest.mock import patch +from packaging.markers import default_environment as packaging_default_environment + from synapse.util.check_dependencies import ( DependencyException, check_requirements, @@ -80,6 +82,22 @@ class TestDependencyChecker(TestCase): ): yield + @contextmanager + def mock_python_version(self, version: str) -> Generator[None, None, None]: + """Override the marker environment to report the supplied `python_version`.""" + + def fake_default_environment() -> dict[str, str]: + env = cast(dict[str, str], dict(packaging_default_environment())) + env["python_version"] = version + env["python_full_version"] = f"{version}.0" + return env + + with patch( + "synapse.util.check_dependencies.default_environment", + side_effect=fake_default_environment, + ): + yield + def test_mandatory_dependency(self) -> None: """Complain if a required package is missing or old.""" with patch( @@ -191,3 +209,35 @@ class TestDependencyChecker(TestCase): with self.mock_installed_package(old): # We also ignore old versions of setuptools_rust check_requirements() + + def test_python_version_markers_respected(self) -> None: + """ + Tests that python_version markers are properly respected. + + Specifically that older versions of dependencies can be installed in + environments with older Python versions. + """ + requirements = [ + "pydantic ~= 2.8; python_version < '3.14'", + "pydantic ~= 2.12; python_version >= '3.14'", + ] + + with patch( + "synapse.util.check_dependencies.metadata.requires", + return_value=requirements, + ): + with self.mock_python_version("3.9"): + with self.mock_installed_package(DummyDistribution("2.12.3")): + check_requirements() + with self.mock_installed_package(DummyDistribution("2.8.1")): + check_requirements() + with self.mock_installed_package(DummyDistribution("2.7.0")): + self.assertRaises(DependencyException, check_requirements) + + with self.mock_python_version("3.14"): + with self.mock_installed_package(DummyDistribution("2.12.3")): + check_requirements() + with self.mock_installed_package(DummyDistribution("2.8.1")): + self.assertRaises(DependencyException, check_requirements) + with self.mock_installed_package(DummyDistribution("2.7.0")): + self.assertRaises(DependencyException, check_requirements)