396de6544a
This PR aims to allow for a clean shutdown of the `SynapseHomeServer` object so that it can be fully deleted and cleaned up by garbage collection without shutting down the entire python process. Fix https://github.com/element-hq/synapse-small-hosts/issues/50 ### Pull Request Checklist <!-- Please read https://element-hq.github.io/synapse/latest/development/contributing_guide.html before submitting your pull request --> * [x] Pull request is based on the develop branch * [x] Pull request includes a [changelog file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog). The entry should: - Be a short description of your change which makes sense to users. "Fixed a bug that prevented receiving messages from other servers." instead of "Moved X method from `EventStore` to `EventWorkerStore`.". - Use markdown where necessary, mostly for `code blocks`. - End with either a period (.) or an exclamation mark (!). - Start with a capital letter. - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry. * [x] [Code style](https://element-hq.github.io/synapse/latest/code_style.html) is correct (run the [linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters)) --------- Co-authored-by: Eric Eastwood <erice@element.io>
194 lines
8.5 KiB
Python
194 lines
8.5 KiB
Python
#
|
|
# This file is licensed under the Affero General Public License (AGPL) version 3.
|
|
#
|
|
# Copyright (C) 2025 New Vector, Ltd
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# See the GNU Affero General Public License for more details:
|
|
# <https://www.gnu.org/licenses/agpl-3.0.html>.
|
|
#
|
|
# Originally licensed under the Apache License, Version 2.0:
|
|
# <http://www.apache.org/licenses/LICENSE-2.0>.
|
|
#
|
|
# [This file includes modifications made by New Vector Limited]
|
|
#
|
|
#
|
|
|
|
import gc
|
|
import weakref
|
|
|
|
from synapse.app.homeserver import SynapseHomeServer
|
|
from synapse.storage.background_updates import UpdaterStatus
|
|
|
|
from tests.server import (
|
|
cleanup_test_reactor_system_event_triggers,
|
|
get_clock,
|
|
setup_test_homeserver,
|
|
)
|
|
from tests.unittest import HomeserverTestCase
|
|
|
|
|
|
class HomeserverCleanShutdownTestCase(HomeserverTestCase):
|
|
def setUp(self) -> None:
|
|
pass
|
|
|
|
# NOTE: ideally we'd have another test to ensure we properly shutdown with
|
|
# real in-flight HTTP requests since those result in additional resources being
|
|
# setup that hold strong references to the homeserver.
|
|
# Mainly, the HTTP channel created by a real TCP connection from client to server
|
|
# is held open between requests and care needs to be taken in Twisted to ensure it is properly
|
|
# closed in a timely manner during shutdown. Simulating this behaviour in a unit test
|
|
# won't be as good as a proper integration test in complement.
|
|
|
|
def test_clean_homeserver_shutdown(self) -> None:
|
|
"""Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected"""
|
|
self.reactor, self.clock = get_clock()
|
|
self.hs = setup_test_homeserver(
|
|
cleanup_func=self.addCleanup,
|
|
reactor=self.reactor,
|
|
homeserver_to_use=SynapseHomeServer,
|
|
clock=self.clock,
|
|
)
|
|
self.wait_for_background_updates()
|
|
|
|
hs_ref = weakref.ref(self.hs)
|
|
|
|
# Run the reactor so any `callWhenRunning` functions can be cleared out.
|
|
self.reactor.run()
|
|
# This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor`
|
|
# we use in tests doesn't handle this properly (see doc comment)
|
|
cleanup_test_reactor_system_event_triggers(self.reactor)
|
|
|
|
# Cleanup the homeserver.
|
|
self.get_success(self.hs.shutdown())
|
|
|
|
# Cleanup the internal reference in our test case
|
|
del self.hs
|
|
|
|
# Force garbage collection.
|
|
gc.collect()
|
|
|
|
# Ensure the `HomeServer` hs been garbage collected by attempting to use the
|
|
# weakref to it.
|
|
if hs_ref() is not None:
|
|
self.fail("HomeServer reference should not be valid at this point")
|
|
|
|
# To help debug this test when it fails, it is useful to leverage the
|
|
# `objgraph` module.
|
|
# The following code serves as an example of what I have found to be useful
|
|
# when tracking down references holding the `SynapseHomeServer` in memory:
|
|
#
|
|
# all_objects = gc.get_objects()
|
|
# for obj in all_objects:
|
|
# try:
|
|
# # These are a subset of types that are typically involved with
|
|
# # holding the `HomeServer` in memory. You may want to inspect
|
|
# # other types as well.
|
|
# if isinstance(obj, DataStore):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# db_obj = obj
|
|
# if isinstance(obj, SynapseHomeServer):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# synapse_hs = obj
|
|
# if isinstance(obj, SynapseSite):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# sysite = obj
|
|
# if isinstance(obj, DatabasePool):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# dbpool = obj
|
|
# except Exception:
|
|
# pass
|
|
#
|
|
# print(sys.getrefcount(hs_ref()), "refs to", hs_ref())
|
|
#
|
|
# # The following values for `max_depth` and `too_many` have been found to
|
|
# # render a useful amount of information without taking an overly long time
|
|
# # to generate the result.
|
|
# objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10)
|
|
|
|
def test_clean_homeserver_shutdown_mid_background_updates(self) -> None:
|
|
"""Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected
|
|
before background updates have completed"""
|
|
self.reactor, self.clock = get_clock()
|
|
self.hs = setup_test_homeserver(
|
|
cleanup_func=self.addCleanup,
|
|
reactor=self.reactor,
|
|
homeserver_to_use=SynapseHomeServer,
|
|
clock=self.clock,
|
|
)
|
|
|
|
# Pump the background updates by a single iteration, just to ensure any extra
|
|
# resources it uses have been started.
|
|
store = weakref.proxy(self.hs.get_datastores().main)
|
|
self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1)
|
|
|
|
hs_ref = weakref.ref(self.hs)
|
|
|
|
# Run the reactor so any `callWhenRunning` functions can be cleared out.
|
|
self.reactor.run()
|
|
# This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor`
|
|
# we use in tests doesn't handle this properly (see doc comment)
|
|
cleanup_test_reactor_system_event_triggers(self.reactor)
|
|
|
|
# Ensure the background updates are not complete.
|
|
self.assertNotEqual(store.db_pool.updates.get_status(), UpdaterStatus.COMPLETE)
|
|
|
|
# Cleanup the homeserver.
|
|
self.get_success(self.hs.shutdown())
|
|
|
|
# Cleanup the internal reference in our test case
|
|
del self.hs
|
|
|
|
# Force garbage collection.
|
|
gc.collect()
|
|
|
|
# Ensure the `HomeServer` hs been garbage collected by attempting to use the
|
|
# weakref to it.
|
|
if hs_ref() is not None:
|
|
self.fail("HomeServer reference should not be valid at this point")
|
|
|
|
# To help debug this test when it fails, it is useful to leverage the
|
|
# `objgraph` module.
|
|
# The following code serves as an example of what I have found to be useful
|
|
# when tracking down references holding the `SynapseHomeServer` in memory:
|
|
#
|
|
# all_objects = gc.get_objects()
|
|
# for obj in all_objects:
|
|
# try:
|
|
# # These are a subset of types that are typically involved with
|
|
# # holding the `HomeServer` in memory. You may want to inspect
|
|
# # other types as well.
|
|
# if isinstance(obj, DataStore):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# db_obj = obj
|
|
# if isinstance(obj, SynapseHomeServer):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# synapse_hs = obj
|
|
# if isinstance(obj, SynapseSite):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# sysite = obj
|
|
# if isinstance(obj, DatabasePool):
|
|
# print(sys.getrefcount(obj), "refs to", obj)
|
|
# if not isinstance(obj, weakref.ProxyType):
|
|
# dbpool = obj
|
|
# except Exception:
|
|
# pass
|
|
#
|
|
# print(sys.getrefcount(hs_ref()), "refs to", hs_ref())
|
|
#
|
|
# # The following values for `max_depth` and `too_many` have been found to
|
|
# # render a useful amount of information without taking an overly long time
|
|
# # to generate the result.
|
|
# objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10)
|