diff --git a/changelog.d/18380.misc b/changelog.d/18380.misc
new file mode 100644
index 000000000..5d6017be2
--- /dev/null
+++ b/changelog.d/18380.misc
@@ -0,0 +1 @@
+Fix a memory leak in `_NotifierUserStream`.
\ No newline at end of file
diff --git a/synapse/notifier.py b/synapse/notifier.py
index 1914d0c91..6190432b8 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -158,6 +158,9 @@ class _NotifierUserStream:
             lst = notifier.room_to_user_streams.get(room, set())
             lst.discard(self)
 
+            if not lst:
+                notifier.room_to_user_streams.pop(room, None)
+
         notifier.user_to_user_stream.pop(self.user_id)
 
     def count_listeners(self) -> int: