From 13ecb7d012daaae3336381a283fc3c967a7dcdc4 Mon Sep 17 00:00:00 2001 From: Thibaut ACKERMANN Date: Tue, 11 Oct 2022 21:57:23 +0200 Subject: [PATCH] Redis memleaks and socketleaks (#1015) Hello, while using the `redis-statsdb` option, I found that coturn is leaking sockets (and memory) in case of redis reconnection. This occurs a lot to me, because in my setup I have a `coturn -> haproxy -> redis` and if all my redis servers are down, HaProxy abruptly close the connection to coturn and coturn reconnects periodically. After some time I can see thousands of pending sockets (`CLOSE_WAIT`) : ``` user@server[11:32:48 UTC]:~$ sudo lsof -i | grep turn turnserve 461797 root 15u IPv4 12856075 0t0 TCP server:3478 (LISTEN) turnserve 461797 root 22u IPv4 12856081 0t0 TCP server:3478 (LISTEN) turnserve 461797 root 23u IPv4 12857384 0t0 UDP server:3478 turnserve 461797 root 24u IPv4 12857385 0t0 UDP server:3478 turnserve 461797 root 36u IPv4 12857390 0t0 TCP server:5766 (LISTEN) turnserve 461797 root 43u IPv4 12856096 0t0 TCP server:10059->haproxy-server:redis (CLOSE_WAIT) turnserve 461797 root 46u IPv4 12857403 0t0 TCP server:10087->haproxy-server:redis (CLOSE_WAIT) turnserve 461797 root 48u IPv4 12856124 0t0 TCP server:53867->haproxy-server:redis (CLOSE_WAIT) turnserve 461797 root 50u IPv4 12857633 0t0 TCP server:53875->haproxy-server:redis (CLOSE_WAIT) turnserve 461797 root 51u IPv4 12856138 0t0 TCP server:53877->haproxy-server:redis (CLOSE_WAIT) turnserve 461797 root 54u IPv4 12857738 0t0 TCP server:10001->haproxy-server:redis (CLOSE_WAIT) turnserve 461797 root 55u IPv4 12856152 0t0 TCP server:10003->haproxy-server:redis (CLOSE_WAIT) .... (many many lines) ``` After searching and using valgrind I found 2 interesting leaks: ``` ... ==460721== 32 bytes in 1 blocks are definitely lost in loss record 586 of 1,053 ==460721== at 0x483877F: malloc (vg_replace_malloc.c:307) ==460721== by 0x1414FF: RyconninfoParse (dbd_redis.c:69) ==460721== by 0x141B04: get_redis_async_connection (dbd_redis.c:169) ==460721== by 0x110D7B: create_ioa_engine (ns_ioalib_engine_impl.c:407) ==460721== by 0x12ECB0: setup_admin_thread (turn_admin_server.c:1309) ==460721== by 0x127724: run_admin_server_thread (netengine.c:1815) ==460721== by 0x4DA9EA6: start_thread (pthread_create.c:477) ==460721== by 0x4EC0AEE: clone (clone.S:95) ... ==460979== 2,170 (688 direct, 1,482 indirect) bytes in 2 blocks are definitely lost in loss record 1,029 of 1,049 ==460979== at 0x483AD7B: realloc (vg_replace_malloc.c:834) ==460979== by 0x49A1BD0: ??? (in /usr/lib/x86_64-linux-gnu/libhiredis.so.0.14) ==460979== by 0x49A2829: redisAsyncConnect (in /usr/lib/x86_64-linux-gnu/libhiredis.so.0.14) ==460979== by 0x13DB37: redis_reconnect (hiredis_libevent2.c:331) ==460979== by 0x13D1A7: redisLibeventReadEvent (hiredis_libevent2.c:101) ==460979== by 0x4D5135E: ??? (in /usr/lib/x86_64-linux-gnu/libevent_core-2.1.so.7.0.1) ==460979== by 0x4D51A9E: event_base_loop (in /usr/lib/x86_64-linux-gnu/libevent_core-2.1.so.7.0.1) ==460979== by 0x126D5A: run_events (netengine.c:1579) ==460979== by 0x127272: run_general_relay_thread (netengine.c:1707) ==460979== by 0x4DA9EA6: start_thread (pthread_create.c:477) ==460979== by 0x4EC0AEE: clone (clone.S:95) ==460979== ... ``` I made 1 commit for each fix. Obviously with these fixes, I don't have anymore the leaks of thousands of sockets (even after some time) Thanks & hope it helps. Thibaut --- src/apps/common/hiredis_libevent2.c | 1 + src/apps/relay/dbdrivers/dbd_redis.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/apps/common/hiredis_libevent2.c b/src/apps/common/hiredis_libevent2.c index 6a100b9..ebfe737 100644 --- a/src/apps/common/hiredis_libevent2.c +++ b/src/apps/common/hiredis_libevent2.c @@ -325,6 +325,7 @@ static void redis_reconnect(struct redisLibeventEvents *e) redisAsyncContext *ac = NULL; if(e->context) { + redisAsyncFree(e->context); e->context = NULL; } diff --git a/src/apps/relay/dbdrivers/dbd_redis.c b/src/apps/relay/dbdrivers/dbd_redis.c index f7250b2..7969164 100644 --- a/src/apps/relay/dbdrivers/dbd_redis.c +++ b/src/apps/relay/dbdrivers/dbd_redis.c @@ -62,6 +62,7 @@ static void RyconninfoFree(Ryconninfo *co) { if(co->dbname) free(co->dbname); if(co->password) free(co->password); memset(co, 0, sizeof(Ryconninfo)); + free(co); } }