master/synapse
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Update check_dependencies to support markers (#19110)

Browse Source
This commit is contained in:
Andrew Morgan 2025-10-30 22:33:29 +01:00 committed by GitHub
parent c0b9437ab6
commit 300c5558ab
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 176 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/19110.misc Normal file
View File

@ -0,0 +1 @@
Allow Synapse's runtime dependency checking code to take packaging markers (i.e. `python <= 3.14`) into account when checking dependencies.

131
synapse/util/check_dependencies.py
View File

@ -28,8 +28,9 @@ require. But this is probably just symptomatic of Python's package management.
import logging
from importlib import metadata
from typing import Iterable, NamedTuple, Optional
from typing import Any, Iterable, NamedTuple, Optional, Sequence, cast
from packaging.markers import Marker, Value, Variable, default_environment
from packaging.requirements import Requirement
DISTRIBUTION_NAME = "matrix-synapse"
@ -65,9 +66,23 @@ RUNTIME_EXTRAS = set(ALL_EXTRAS) - DEV_EXTRAS
VERSION = metadata.version(DISTRIBUTION_NAME)
def _marker_environment(extra: str) -> dict[str, str]:
"""Return the marker environment for `extra`, seeded with the current interpreter."""
env = cast(dict[str, str], dict(default_environment()))
env["extra"] = extra
return env
def _is_dev_dependency(req: Requirement) -> bool:
return req.marker is not None and any(
req.marker.evaluate({"extra": e}) for e in DEV_EXTRAS
"""Return True if `req` is a development dependency."""
if req.marker is None:
return False
marker_extras = _extras_from_marker(req.marker)
return any(
extra in DEV_EXTRAS and req.marker.evaluate(_marker_environment(extra))
for extra in marker_extras
)
@ -95,6 +110,7 @@ def _generic_dependencies() -> Iterable[Dependency]:
"""Yield pairs (requirement, must_be_installed)."""
requirements = metadata.requires(DISTRIBUTION_NAME)
assert requirements is not None
env_no_extra = _marker_environment("")
for raw_requirement in requirements:
req = Requirement(raw_requirement)
if _is_dev_dependency(req) or _should_ignore_runtime_requirement(req):
@ -103,7 +119,7 @@ def _generic_dependencies() -> Iterable[Dependency]:
# https://packaging.pypa.io/en/latest/markers.html#usage notes that
# > Evaluating an extra marker with no environment is an error
# so we pass in a dummy empty extra value here.
must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""})
must_be_installed = req.marker is None or req.marker.evaluate(env_no_extra)
yield Dependency(req, must_be_installed)
@ -111,6 +127,8 @@ def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
"""Yield additional dependencies needed for a given `extra`."""
requirements = metadata.requires(DISTRIBUTION_NAME)
assert requirements is not None
env_no_extra = _marker_environment("")
env_for_extra = _marker_environment(extra)
for raw_requirement in requirements:
req = Requirement(raw_requirement)
if _is_dev_dependency(req):
@ -118,12 +136,84 @@ def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
# Exclude mandatory deps by only selecting deps needed with this extra.
if (
req.marker is not None
and req.marker.evaluate({"extra": extra})
and not req.marker.evaluate({"extra": ""})
and req.marker.evaluate(env_for_extra)
and not req.marker.evaluate(env_no_extra)
):
yield Dependency(req, True)
def _values_from_marker_value(value: Value) -> set[str]:
"""Extract text values contained in a marker `Value`."""
raw: Any = value.value
if isinstance(raw, str):
return {raw}
if isinstance(raw, (tuple, list)):
return {str(item) for item in raw}
return {str(raw)}
def _extras_from_marker(marker: Optional[Marker]) -> set[str]:
"""Return every `extra` referenced in the supplied marker tree."""
extras: set[str] = set()
if marker is None:
return extras
def collect(tree: object) -> None:
if isinstance(tree, list):
for item in tree:
collect(item)
elif isinstance(tree, tuple) and len(tree) == 3:
lhs, _op, rhs = tree
if (
isinstance(lhs, Variable)
and lhs.value == "extra"
and isinstance(rhs, Value)
):
extras.update(_values_from_marker_value(rhs))
elif (
isinstance(rhs, Variable)
and rhs.value == "extra"
and isinstance(lhs, Value)
):
extras.update(_values_from_marker_value(lhs))
collect(marker._markers)
return extras
def _extras_to_consider_for_requirement(
marker: Marker, base_candidates: Sequence[str]
) -> set[str]:
"""
Augment `base_candidates` with extras explicitly mentioned in `marker`.
Markers can mention extras (e.g. `extra == "saml2"`).
"""
# Avoid modifying the input sequence.
# Use a set to efficiently avoid duplicate extras.
extras = set(base_candidates)
for candidate in _extras_from_marker(marker):
extras.add(candidate)
return extras
def _marker_applies_for_any_extra(requirement: Requirement, extras: set[str]) -> bool:
"""Check whether a requirement's marker matches any evaluated `extra`."""
if requirement.marker is None:
return True
return any(
requirement.marker.evaluate(_marker_environment(extra)) for extra in extras
)
def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str:
if extra:
return (
@ -164,7 +254,7 @@ def _no_reported_version(requirement: Requirement, extra: Optional[str] = None)
def check_requirements(extra: Optional[str] = None) -> None:
"""Check Synapse's dependencies are present and correctly versioned.
If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in
If provided, `extra` must be the name of an packaging extra (e.g. "saml2" in
`pip install matrix-synapse[saml2]`).
If `extra` is None, this function checks that
@ -174,6 +264,15 @@ def check_requirements(extra: Optional[str] = None) -> None:
If `extra` is not None, this function checks that
- the dependencies needed for that extra are installed and correctly versioned.
`marker`s are optional attributes on each requirement which specify
conditions under which the requirement applies. For example, a requirement
might only be needed on Windows, or with Python < 3.14. Markers can
additionally mention `extras` themselves, meaning a requirement may not
apply if the marker mentions an extra that the user has not asked for.
This function skips a requirement when its markers do not apply in the
current environment.
:raises DependencyException: if a dependency is missing or incorrectly versioned.
:raises ValueError: if this extra does not exist.
"""
@ -188,7 +287,25 @@ def check_requirements(extra: Optional[str] = None) -> None:
deps_unfulfilled = []
errors = []
if extra is None:
# Default to all mandatory dependencies (non-dev extras).
# "" means all dependencies that aren't conditional on an extra.
base_extra_candidates: Sequence[str] = ("", *RUNTIME_EXTRAS)
else:
base_extra_candidates = (extra,)
for requirement, must_be_installed in dependencies:
if requirement.marker is not None:
candidate_extras = _extras_to_consider_for_requirement(
requirement.marker, base_extra_candidates
)
# Skip checking this dependency if the requirement's marker object
# (i.e. `python_version < "3.14" and os_name == "win32"`) does not
# apply for any of the extras we're considering.
if not _marker_applies_for_any_extra(requirement, candidate_extras):
continue
# Check if the requirement is installed and correctly versioned.
try:
dist: metadata.Distribution = metadata.distribution(requirement.name)
except metadata.PackageNotFoundError:

52
tests/util/test_check_dependencies.py
View File

@ -22,9 +22,11 @@
from contextlib import contextmanager
from os import PathLike
from pathlib import Path
from typing import Generator, Optional, Union
from typing import Generator, Optional, Union, cast
from unittest.mock import patch
from packaging.markers import default_environment as packaging_default_environment
from synapse.util.check_dependencies import (
DependencyException,
check_requirements,
@ -80,6 +82,22 @@ class TestDependencyChecker(TestCase):
):
yield
@contextmanager
def mock_python_version(self, version: str) -> Generator[None, None, None]:
"""Override the marker environment to report the supplied `python_version`."""
def fake_default_environment() -> dict[str, str]:
env = cast(dict[str, str], dict(packaging_default_environment()))
env["python_version"] = version
env["python_full_version"] = f"{version}.0"
return env
with patch(
"synapse.util.check_dependencies.default_environment",
side_effect=fake_default_environment,
):
yield
def test_mandatory_dependency(self) -> None:
"""Complain if a required package is missing or old."""
with patch(
@ -191,3 +209,35 @@ class TestDependencyChecker(TestCase):
with self.mock_installed_package(old):
# We also ignore old versions of setuptools_rust
check_requirements()
def test_python_version_markers_respected(self) -> None:
"""
Tests that python_version markers are properly respected.
Specifically that older versions of dependencies can be installed in
environments with older Python versions.
"""
requirements = [
"pydantic ~= 2.8; python_version < '3.14'",
"pydantic ~= 2.12; python_version >= '3.14'",
]
with patch(
"synapse.util.check_dependencies.metadata.requires",
return_value=requirements,
):
with self.mock_python_version("3.9"):
with self.mock_installed_package(DummyDistribution("2.12.3")):
check_requirements()
with self.mock_installed_package(DummyDistribution("2.8.1")):
check_requirements()
with self.mock_installed_package(DummyDistribution("2.7.0")):
self.assertRaises(DependencyException, check_requirements)
with self.mock_python_version("3.14"):
with self.mock_installed_package(DummyDistribution("2.12.3")):
check_requirements()
with self.mock_installed_package(DummyDistribution("2.8.1")):
self.assertRaises(DependencyException, check_requirements)
with self.mock_installed_package(DummyDistribution("2.7.0")):
self.assertRaises(DependencyException, check_requirements)