From 687f3451bde8e954af7c51dc9cea52e53de84e32 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 13:44:05 +0100 Subject: [PATCH 01/12] 0.27.3 --- CHANGES.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 1372de424..9f974e687 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,49 @@ +Changes in synapse v0.27.3 (2018-04-09) +======================================= + +Notable changes include API support for joinability of groups. Also new metrics +and phone home stats. Phone home stats include better visibility of system usage +so we can tweak synpase to work better for all users rather than our own experience +with matrix.org. Also, recording 'r30' stat which is the measure we use to track +overal growth of the Matrix ecosystem. It is defined as:- + "Counts the number of native 30 day retained users, defined as:- + * Users who have created their accounts more than 30 days + * Where last seen at most 30 days ago + * Where account creation and last_seen are > 30 days" + + +Features +* Add joinability for groups (PR #3045) +* Implement group join API (PR #3046) +* Add counter metrics for calculating state delta (PR #3033) +* R30 stats (PR #3041) +* Measure time it takes to calculate state group ID (PR #3043) +* Add basic performance statistics to phone home (PR #3044) +* Add response size metrics (PR #3071) +* phone home cache size configurations (PR #3063) + +Changes +* Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live! +* Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile! +* Use .iter* to avoid copies in StateHandler (PR #3006) +* Linearize calls to _generate_user_id (PR #3029) +* Remove last usage of ujson (PR #3030) +* Use simplejson throughout (PR #3048) +* Use static JSONEncoders (PR #3049) +* Remove uses of events.content (PR #3060) +* Improve database cache performance (PR #3068) + +Bug fixes +* Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte! +* Fix replication after switch to simplejson (PR #3015) +* Fix replication after switch to simplejson (PR #3015) +* 404 correctly on missing paths via NoResource (PR #3022) +* Fix error when claiming e2e keys from offline servers (PR #3034) +* fix tests/storage/test_user_directory.py (PR #3042) +* use PUT instead of POST for federating groups/m.join_policy (PR #3070) Thanks to @krombel! +* postgres port script: fix state_groups_pkey error (PR #3072) + + Changes in synapse v0.27.2 (2018-03-26) ======================================= From 072fb594465126a523126b7607d19fb2f190e01a Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 13:49:25 +0100 Subject: [PATCH 02/12] bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index a9d5198ab..1fa2de526 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.2" +__version__ = "0.27.3" From c115deed12746f08ccc6ef7294cc9649c99aa1b7 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 15:54:32 +0100 Subject: [PATCH 03/12] Update CHANGES.rst --- CHANGES.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index df469a993..23ee926f3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,7 +12,8 @@ overal growth of the Matrix ecosystem. It is defined as:- * Where account creation and last_seen are > 30 days" -Features +Features: + * Add joinability for groups (PR #3045) * Implement group join API (PR #3046) * Add counter metrics for calculating state delta (PR #3033) @@ -22,7 +23,8 @@ Features * Add response size metrics (PR #3071) * phone home cache size configurations (PR #3063) -Changes +Changes: + * Add a blurb explaining the main synapse worker (PR #2886) Thanks to @turt2live! * Replace old style error catching with 'as' keyword (PR #3000) Thanks to @NotAFile! * Use .iter* to avoid copies in StateHandler (PR #3006) @@ -33,7 +35,8 @@ Changes * Remove uses of events.content (PR #3060) * Improve database cache performance (PR #3068) -Bug fixes +Bug fixes: + * Add room_id to the response of `rooms/{roomId}/join` (PR #2986) Thanks to @jplatte! * Fix replication after switch to simplejson (PR #3015) * Fix replication after switch to simplejson (PR #3015) From 21d5a2a08e053bf69082dd0c7142e9b299cee0b0 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 15:55:41 +0100 Subject: [PATCH 04/12] Update CHANGES.rst --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 23ee926f3..e8ff469a1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,8 @@ and phone home stats. Phone home stats include better visibility of system usage so we can tweak synpase to work better for all users rather than our own experience with matrix.org. Also, recording 'r30' stat which is the measure we use to track overal growth of the Matrix ecosystem. It is defined as:- - "Counts the number of native 30 day retained users, defined as:- + +Counts the number of native 30 day retained users, defined as:- * Users who have created their accounts more than 30 days * Where last seen at most 30 days ago * Where account creation and last_seen are > 30 days" From d9ae2bc826910c5e70766fb70f0d488faf350fe7 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 16:00:31 +0100 Subject: [PATCH 05/12] bump version to release candidate --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 1fa2de526..354415e5c 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.3" +__version__ = "0.27.3-rc1" From b151eb14a2c680467ac0ff6198a016cba3156fd2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 16:01:59 +0100 Subject: [PATCH 06/12] Update CHANGES.rst --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index e8ff469a1..d80756dc7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.27.3 (2018-04-09) +Changes in synapse v0.27.3-rc1 (2018-04-09) ======================================= Notable changes include API support for joinability of groups. Also new metrics From 5232d3bfb1168a00c136c62359bad4f0fbec0ae2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 17:25:57 +0100 Subject: [PATCH 07/12] version bump v0.27.3-rc2 --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 354415e5c..7f6090baf 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.27.3-rc1" +__version__ = "0.27.3-rc2" From 64bc2162ef1c51cf5d5baa3cd3b303216b55df8c Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 17:50:36 +0100 Subject: [PATCH 08/12] Fix psycopg2 interpolation --- synapse/storage/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 4800584b5..01c580b66 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -289,11 +289,11 @@ class DataStore(RoomMemberStore, RoomStore, user_id, last_seen, CASE - WHEN user_agent LIKE '%Android%' THEN 'android' - WHEN user_agent LIKE '%iOS%' THEN 'ios' - WHEN user_agent LIKE '%Electron%' THEN 'electron' - WHEN user_agent LIKE '%Mozilla%' THEN 'web' - WHEN user_agent LIKE '%Gecko%' THEN 'web' + WHEN user_agent LIKE '%%Android%%' THEN 'android' + WHEN user_agent LIKE '%%iOS%%' THEN 'ios' + WHEN user_agent LIKE '%%Electron%%' THEN 'electron' + WHEN user_agent LIKE '%%Mozilla%%' THEN 'web' + WHEN user_agent LIKE '%%Gecko%%' THEN 'web' ELSE 'unknown' END AS platform From 9a311adfea940984f6a007a97afec39f23cb1be2 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 17:52:08 +0100 Subject: [PATCH 09/12] v0.27.3-rc2 --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index d80756dc7..684c5c788 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +Changes in synapse v0.27.3-rc2 (2018-04-09) +========================================== +v0.27.3-rc1 used a stale version of the develop branch so the changelog overstates +the functionality. v0.27.3-rc2 is up to date, rc1 should be ignored. + Changes in synapse v0.27.3-rc1 (2018-04-09) ======================================= From 87770300d590eb105694e1cbc24aa798f788ca52 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 18:38:59 +0100 Subject: [PATCH 10/12] Fix msec to sec --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 01c580b66..e736a857e 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -276,7 +276,7 @@ class DataStore(RoomMemberStore, RoomStore, def _count_r30_users(txn): thirty_days_in_secs = 86400 * 30 now = int(self._clock.time_msec()) - thirty_days_ago_in_secs = now - thirty_days_in_secs + thirty_days_ago_in_secs = now/1000 - thirty_days_in_secs sql = """ SELECT platform, COALESCE(count(*), 0) FROM ( From 61b439c90437527afdfa6b46e5ab88de6b074792 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 18:43:48 +0100 Subject: [PATCH 11/12] Fix msec to sec, again --- synapse/storage/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index e736a857e..417b9aa2a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -275,9 +275,9 @@ class DataStore(RoomMemberStore, RoomStore, """ def _count_r30_users(txn): thirty_days_in_secs = 86400 * 30 - now = int(self._clock.time_msec()) - thirty_days_ago_in_secs = now/1000 - thirty_days_in_secs - + now = int(self._clock.time()) + thirty_days_ago_in_secs = now - thirty_days_in_secs + print str(thirty_days_ago_in_secs) sql = """ SELECT platform, COALESCE(count(*), 0) FROM ( SELECT From 41e06118954234878311cea3e7a55a88e6eec784 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Mon, 9 Apr 2018 18:44:20 +0100 Subject: [PATCH 12/12] remove errant print --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 417b9aa2a..eacd49d6a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -277,7 +277,7 @@ class DataStore(RoomMemberStore, RoomStore, thirty_days_in_secs = 86400 * 30 now = int(self._clock.time()) thirty_days_ago_in_secs = now - thirty_days_in_secs - print str(thirty_days_ago_in_secs) + sql = """ SELECT platform, COALESCE(count(*), 0) FROM ( SELECT