From c938795576b098a4288a1f4025e59a689ed64701 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Fri, 22 Jan 2021 17:17:42 +0100 Subject: [PATCH 01/17] Read init sync to a file and split into smaller files to handle it --- CHANGES.md | 1 + ...ventInsertType.java => EventInsertType.kt} | 4 +- .../android/sdk/internal/di/MoshiProvider.kt | 2 + .../notification/ProcessEventForPushTask.kt | 4 +- .../sync/InitialSyncStatusRepository.kt | 111 +++++++++++++ .../session/sync/InitialSyncStrategy.kt | 53 ++++++ .../internal/session/sync/RoomSyncHandler.kt | 70 ++++++-- .../sdk/internal/session/sync/SyncAPI.kt | 13 ++ .../sdk/internal/session/sync/SyncTask.kt | 152 +++++++++++++++++- .../session/sync/model/LazyRoomSync.kt | 44 +++++ .../sync/model/LazyRoomSyncJsonAdapter.kt | 90 +++++++++++ .../session/sync/model/RoomsSyncResponse.kt | 2 +- .../android/sdk/internal/util/LogUtil.kt | 29 ++++ .../src/main/res/values/strings.xml | 2 + .../vector/app/features/home/HomeActivity.kt | 18 +++ vector/src/main/res/menu/home.xml | 13 +- 16 files changed, 583 insertions(+), 25 deletions(-) rename matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/{EventInsertType.java => EventInsertType.kt} (88%) create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStatusRepository.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStrategy.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt diff --git a/CHANGES.md b/CHANGES.md index 46057a37ec..930068dd6c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -90,6 +90,7 @@ Improvements 🙌: - SSO support for cross signing (#1062) - Deactivate account when logged in with SSO (#1264) - SSO UIA doesn't work (#2754) + - Improve initial sync performance (#983) Bugfix 🐛: - Fix clear cache issue: sometimes, after a clear cache, there is still a token, so the init sync service is not started. diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventInsertType.java b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventInsertType.kt similarity index 88% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventInsertType.java rename to matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventInsertType.kt index 05153c5734..463ccb2f46 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventInsertType.java +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/database/model/EventInsertType.kt @@ -14,9 +14,9 @@ * limitations under the License. */ -package org.matrix.android.sdk.internal.database.model; +package org.matrix.android.sdk.internal.database.model -public enum EventInsertType { +internal enum class EventInsertType { INITIAL_SYNC, INCREMENTAL_SYNC, PAGINATION, diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt index 48fa41b350..c0595a9273 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt @@ -36,6 +36,7 @@ import org.matrix.android.sdk.internal.network.parsing.ForceToBooleanJsonAdapter import org.matrix.android.sdk.internal.network.parsing.RuntimeJsonAdapterFactory import org.matrix.android.sdk.internal.network.parsing.TlsVersionMoshiAdapter import org.matrix.android.sdk.internal.network.parsing.UriMoshiAdapter +import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter object MoshiProvider { @@ -44,6 +45,7 @@ object MoshiProvider { .add(ForceToBooleanJsonAdapter()) .add(CipherSuiteMoshiAdapter()) .add(TlsVersionMoshiAdapter()) + .add(LazyRoomSyncJsonAdapter()) .add(RuntimeJsonAdapterFactory.of(MessageContent::class.java, "msgtype", MessageDefaultContent::class.java) .registerSubtype(MessageTextContent::class.java, MessageType.MSGTYPE_TEXT) .registerSubtype(MessageNoticeContent::class.java, MessageType.MSGTYPE_NOTICE) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/notification/ProcessEventForPushTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/notification/ProcessEventForPushTask.kt index 7763251a01..4754265c49 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/notification/ProcessEventForPushTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/notification/ProcessEventForPushTask.kt @@ -50,7 +50,7 @@ internal class DefaultProcessEventForPushTask @Inject constructor( } val newJoinEvents = params.syncResponse.join .mapNotNull { (key, value) -> - value.timeline?.events?.map { it.copy(roomId = key) } + value.roomSync.timeline?.events?.map { it.copy(roomId = key) } } .flatten() val inviteEvents = params.syncResponse.invite @@ -80,7 +80,7 @@ internal class DefaultProcessEventForPushTask @Inject constructor( val allRedactedEvents = params.syncResponse.join .asSequence() - .mapNotNull { (_, value) -> value.timeline?.events } + .mapNotNull { (_, value) -> value.roomSync.timeline?.events } .flatten() .filter { it.type == EventType.REDACTION } .mapNotNull { it.redacts } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStatusRepository.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStatusRepository.kt new file mode 100644 index 0000000000..4b82ecc3e5 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStatusRepository.kt @@ -0,0 +1,111 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync + +import com.squareup.moshi.JsonClass +import okio.buffer +import okio.source +import org.matrix.android.sdk.internal.di.MoshiProvider +import timber.log.Timber +import java.io.File + +@JsonClass(generateAdapter = true) +internal data class InitialSyncStatus( + val step: Int = STEP_INIT, + val downloadedDate: Long = 0 +) { + companion object { + const val STEP_INIT = 0 + const val STEP_DOWNLOADING = 1 + const val STEP_DOWNLOADED = 2 + const val STEP_PARSED = 3 + const val STEP_SUCCESS = 4 + } +} + +internal interface InitialSyncStatusRepository { + fun getStep(): Int + + fun setStep(step: Int) +} + +/** + * This class handle the current status of an initial sync and persist it on the disk, to be robust against crash + */ +internal class FileInitialSyncStatusRepository(directory: File) : InitialSyncStatusRepository { + + companion object { + // After 2 hours, we consider that the downloaded file is outdated: + // - if a problem occurs, it's for big accounts, and big accounts have lots of new events in 2 hours + // - For small accounts, there should be no problem, so 2 hours delay will never be used. + private const val INIT_SYNC_FILE_LIFETIME = 2 * 60 * 60 * 1_000L + } + + private val file = File(directory, "status.json") + private val jsonAdapter = MoshiProvider.providesMoshi().adapter(InitialSyncStatus::class.java) + + private var cache: InitialSyncStatus? = null + + override fun getStep(): Int { + ensureCache() + val state = cache?.step ?: InitialSyncStatus.STEP_INIT + return if (state >= InitialSyncStatus.STEP_DOWNLOADED + && System.currentTimeMillis() > (cache?.downloadedDate ?: 0) + INIT_SYNC_FILE_LIFETIME) { + Timber.v("INIT_SYNC downloaded file is outdated, download it again") + // The downloaded file is outdated + setStep(InitialSyncStatus.STEP_INIT) + InitialSyncStatus.STEP_INIT + } else { + state + } + } + + override fun setStep(step: Int) { + var newStatus = cache?.copy(step = step) ?: InitialSyncStatus(step = step) + if (step == InitialSyncStatus.STEP_DOWNLOADED) { + // Also store the downloaded date + newStatus = newStatus.copy( + downloadedDate = System.currentTimeMillis() + ) + } + cache = newStatus + writeFile() + } + + private fun ensureCache() { + if (cache == null) readFile() + } + + /** + * File -> Cache + */ + private fun readFile() { + cache = file + .takeIf { it.exists() } + ?.let { jsonAdapter.fromJson(it.source().buffer()) } + } + + /** + * Cache -> File + */ + private fun writeFile() { + file.delete() + cache + ?.let { jsonAdapter.toJson(it) } + ?.let { file.writeText(it) } + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStrategy.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStrategy.kt new file mode 100644 index 0000000000..fca92870ca --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/InitialSyncStrategy.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync + +var initialSyncStrategy: InitialSyncStrategy = InitialSyncStrategy.Optimized() + +sealed class InitialSyncStrategy { + /** + * Parse the result in its entirety + * Pros: + * - Faster to handle parsed data + * Cons: + * - Slower to download and parse data + * - big RAM usage + * - not robust to crash + */ + object Legacy : InitialSyncStrategy() + + /** + * Optimized. + * First store the request result in a file, to avoid doing it again in case of crash + */ + data class Optimized( + /** + * Limit to reach to decide to split the init sync response into smaller files + * Empiric value: 1 megabytes + */ + val minSizeToSplit: Long = 1024 * 1024, + /** + * Limit per room to reach to decide to store a join room into a file + * Empiric value: 10 kilobytes + */ + val minSizeToStoreInFile: Long = 10 * 1024, + /** + * Max number of rooms to insert at a time in database (to avoid too much RAM usage) + */ + val maxRoomsToInsert: Int = 100 + ) : InitialSyncStrategy() +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt index 6d1b3ae034..979c2888d3 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt @@ -51,6 +51,7 @@ import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.extensions.clearWith import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService import org.matrix.android.sdk.internal.session.mapWithProgress +import org.matrix.android.sdk.internal.session.reportSubtask import org.matrix.android.sdk.internal.session.room.membership.RoomChangeMembershipStateDataSource import org.matrix.android.sdk.internal.session.room.membership.RoomMemberEventHandler import org.matrix.android.sdk.internal.session.room.read.FullyReadContent @@ -59,12 +60,14 @@ import org.matrix.android.sdk.internal.session.room.timeline.PaginationDirection import org.matrix.android.sdk.internal.session.room.timeline.TimelineInput import org.matrix.android.sdk.internal.session.room.typing.TypingEventContent import org.matrix.android.sdk.internal.session.sync.model.InvitedRoomSync +import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync import org.matrix.android.sdk.internal.session.sync.model.RoomSync import org.matrix.android.sdk.internal.session.sync.model.RoomSyncAccountData import org.matrix.android.sdk.internal.session.sync.model.RoomSyncEphemeral import org.matrix.android.sdk.internal.session.sync.model.RoomsSyncResponse import timber.log.Timber import javax.inject.Inject +import kotlin.math.ceil internal class RoomSyncHandler @Inject constructor(private val readReceiptHandler: ReadReceiptHandler, private val roomSummaryUpdater: RoomSummaryUpdater, @@ -78,7 +81,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle private val timelineInput: TimelineInput) { sealed class HandlingStrategy { - data class JOINED(val data: Map) : HandlingStrategy() + data class JOINED(val data: Map) : HandlingStrategy() data class INVITED(val data: Map) : HandlingStrategy() data class LEFT(val data: Map) : HandlingStrategy() } @@ -105,10 +108,17 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle } val syncLocalTimeStampMillis = System.currentTimeMillis() val rooms = when (handlingStrategy) { - is HandlingStrategy.JOINED -> - handlingStrategy.data.mapWithProgress(reporter, R.string.initial_sync_start_importing_account_joined_rooms, 0.6f) { - handleJoinedRoom(realm, it.key, it.value, isInitialSync, insertType, syncLocalTimeStampMillis) + is HandlingStrategy.JOINED -> { + if (isInitialSync && initialSyncStrategy is InitialSyncStrategy.Optimized) { + insertJoinRooms(realm, handlingStrategy, insertType, syncLocalTimeStampMillis, reporter) + // Rooms are already inserted, return an empty list + emptyList() + } else { + handlingStrategy.data.mapWithProgress(reporter, R.string.initial_sync_start_importing_account_joined_rooms, 0.6f) { + handleJoinedRoom(realm, it.key, it.value.roomSync, insertType, syncLocalTimeStampMillis) + } } + } is HandlingStrategy.INVITED -> handlingStrategy.data.mapWithProgress(reporter, R.string.initial_sync_start_importing_account_invited_rooms, 0.1f) { handleInvitedRoom(realm, it.key, it.value, insertType, syncLocalTimeStampMillis) @@ -123,17 +133,57 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle realm.insertOrUpdate(rooms) } + private fun insertJoinRooms(realm: Realm, + handlingStrategy: HandlingStrategy.JOINED, + insertType: EventInsertType, + syncLocalTimeStampMillis: Long, + reporter: DefaultInitialSyncProgressService?) { + val maxSize = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.maxRoomsToInsert ?: Int.MAX_VALUE + val listSize = handlingStrategy.data.keys.size + val numberOfChunks = ceil(listSize / maxSize.toDouble()).toInt() + + if (numberOfChunks > 1) { + reportSubtask(reporter, R.string.initial_sync_start_importing_account_joined_rooms, numberOfChunks, 0.6f) { + val chunkSize = listSize / numberOfChunks + Timber.v("INIT_SYNC $listSize rooms to insert, split into $numberOfChunks sublists of $chunkSize items") + // I cannot find a better way to chunk a map, so chunk the keys and then create new maps + handlingStrategy.data.keys + .chunked(chunkSize) + .forEachIndexed { index, roomIds -> + val roomEntities = roomIds + .also { Timber.v("INIT_SYNC insert ${roomIds.size} rooms") } + .map { + handleJoinedRoom( + realm, + it, + (handlingStrategy.data[it] ?: error("Should not happen")).roomSync, + insertType, + syncLocalTimeStampMillis + ) + } + realm.insertOrUpdate(roomEntities) + reporter?.reportProgress(index + 1) + } + } + } else { + // No need to split + val rooms = handlingStrategy.data.mapWithProgress(reporter, R.string.initial_sync_start_importing_account_joined_rooms, 0.6f) { + handleJoinedRoom(realm, it.key, it.value.roomSync, insertType, syncLocalTimeStampMillis) + } + realm.insertOrUpdate(rooms) + } + } + private fun handleJoinedRoom(realm: Realm, roomId: String, roomSync: RoomSync, - isInitialSync: Boolean, insertType: EventInsertType, syncLocalTimestampMillis: Long): RoomEntity { Timber.v("Handle join sync for room $roomId") var ephemeralResult: EphemeralResult? = null if (roomSync.ephemeral?.events?.isNotEmpty() == true) { - ephemeralResult = handleEphemeral(realm, roomId, roomSync.ephemeral, isInitialSync) + ephemeralResult = handleEphemeral(realm, roomId, roomSync.ephemeral, insertType == EventInsertType.INITIAL_SYNC) } if (roomSync.accountData?.events?.isNotEmpty() == true) { @@ -173,8 +223,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle roomSync.timeline.prevToken, roomSync.timeline.limited, insertType, - syncLocalTimestampMillis, - isInitialSync + syncLocalTimestampMillis ) roomEntity.addIfNecessary(chunkEntity) } @@ -278,8 +327,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle prevToken: String? = null, isLimited: Boolean = true, insertType: EventInsertType, - syncLocalTimestampMillis: Long, - isInitialSync: Boolean): ChunkEntity { + syncLocalTimestampMillis: Long): ChunkEntity { val lastChunk = ChunkEntity.findLastForwardChunkOfRoom(realm, roomEntity.roomId) val chunkEntity = if (!isLimited && lastChunk != null) { lastChunk @@ -299,7 +347,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle } eventIds.add(event.eventId) - if (event.isEncrypted() && !isInitialSync) { + if (event.isEncrypted() && insertType != EventInsertType.INITIAL_SYNC) { decryptIfNeeded(event, roomId) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncAPI.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncAPI.kt index 77289f04b4..8e3523bc57 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncAPI.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncAPI.kt @@ -16,6 +16,7 @@ package org.matrix.android.sdk.internal.session.sync +import okhttp3.ResponseBody import org.matrix.android.sdk.internal.network.NetworkConstants import org.matrix.android.sdk.internal.network.TimeOutInterceptor import org.matrix.android.sdk.internal.session.sync.model.SyncResponse @@ -23,6 +24,7 @@ import retrofit2.Call import retrofit2.http.GET import retrofit2.http.Header import retrofit2.http.QueryMap +import retrofit2.http.Streaming internal interface SyncAPI { /** @@ -34,4 +36,15 @@ internal interface SyncAPI { @Header(TimeOutInterceptor.READ_TIMEOUT) readTimeOut: Long = TimeOutInterceptor.DEFAULT_LONG_TIMEOUT, @Header(TimeOutInterceptor.WRITE_TIMEOUT) writeTimeOut: Long = TimeOutInterceptor.DEFAULT_LONG_TIMEOUT ): Call + + /** + * Set all the timeouts to 1 minute by default + */ + @Streaming + @GET(NetworkConstants.URI_API_PREFIX_PATH_R0 + "sync") + fun syncStream(@QueryMap params: Map, + @Header(TimeOutInterceptor.CONNECT_TIMEOUT) connectTimeOut: Long = TimeOutInterceptor.DEFAULT_LONG_TIMEOUT, + @Header(TimeOutInterceptor.READ_TIMEOUT) readTimeOut: Long = TimeOutInterceptor.DEFAULT_LONG_TIMEOUT, + @Header(TimeOutInterceptor.WRITE_TIMEOUT) writeTimeOut: Long = TimeOutInterceptor.DEFAULT_LONG_TIMEOUT + ): Call } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt index bfe3799771..ef766f6d42 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt @@ -16,18 +16,32 @@ package org.matrix.android.sdk.internal.session.sync +import okhttp3.ResponseBody +import okio.buffer +import okio.source import org.matrix.android.sdk.R +import org.matrix.android.sdk.internal.di.MoshiProvider +import org.matrix.android.sdk.internal.di.SessionFilesDirectory import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.network.GlobalErrorReceiver import org.matrix.android.sdk.internal.network.TimeOutInterceptor import org.matrix.android.sdk.internal.network.executeRequest +import org.matrix.android.sdk.internal.network.toFailure import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService import org.matrix.android.sdk.internal.session.filter.FilterRepository import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask +import org.matrix.android.sdk.internal.session.reportSubtask +import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync +import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter import org.matrix.android.sdk.internal.session.sync.model.SyncResponse import org.matrix.android.sdk.internal.session.user.UserStore import org.matrix.android.sdk.internal.task.Task +import org.matrix.android.sdk.internal.util.logDuration +import retrofit2.Response +import retrofit2.awaitResponse import timber.log.Timber +import java.io.File +import java.net.SocketTimeoutException import javax.inject.Inject internal interface SyncTask : Task { @@ -48,9 +62,14 @@ internal class DefaultSyncTask @Inject constructor( private val getHomeServerCapabilitiesTask: GetHomeServerCapabilitiesTask, private val userStore: UserStore, private val syncTaskSequencer: SyncTaskSequencer, - private val globalErrorReceiver: GlobalErrorReceiver + private val globalErrorReceiver: GlobalErrorReceiver, + @SessionFilesDirectory + private val fileDirectory: File ) : SyncTask { + private val workingDir = File(fileDirectory, "is") + private val initialSyncStatusRepository: InitialSyncStatusRepository = FileInitialSyncStatusRepository(workingDir) + override suspend fun execute(params: SyncTask.Params) = syncTaskSequencer.post { doSync(params) } @@ -81,20 +100,137 @@ internal class DefaultSyncTask @Inject constructor( val readTimeOut = (params.timeout + TIMEOUT_MARGIN).coerceAtLeast(TimeOutInterceptor.DEFAULT_LONG_TIMEOUT) - val syncResponse = executeRequest(globalErrorReceiver) { - apiCall = syncAPI.sync( - params = requestParams, - readTimeOut = readTimeOut - ) - } - syncResponseHandler.handleResponse(syncResponse, token) if (isInitialSync) { + logDuration("INIT_SYNC strategy: $initialSyncStrategy") { + if (initialSyncStrategy is InitialSyncStrategy.Optimized) { + safeInitialSync(requestParams) + } else { + val syncResponse = logDuration("INIT_SYNC Request") { + executeRequest(globalErrorReceiver) { + apiCall = syncAPI.sync( + params = requestParams, + readTimeOut = readTimeOut + ) + } + } + + logDuration("INIT_SYNC Database insertion") { + syncResponseHandler.handleResponse(syncResponse, token) + } + } + } initialSyncProgressService.endAll() + } else { + val syncResponse = executeRequest(globalErrorReceiver) { + apiCall = syncAPI.sync( + params = requestParams, + readTimeOut = readTimeOut + ) + } + syncResponseHandler.handleResponse(syncResponse, token) } Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}") } + private suspend fun safeInitialSync(requestParams: Map) { + workingDir.mkdirs() + val workingFile = File(workingDir, "initSync.json") + val status = initialSyncStatusRepository.getStep() + if (workingFile.exists() && status >= InitialSyncStatus.STEP_DOWNLOADED) { + // Go directly to the parse step + Timber.v("INIT_SYNC file is already here") + } else { + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADING) + val syncResponse = logDuration("INIT_SYNC Perform server request") { + reportSubtask(initialSyncProgressService, R.string.initial_sync_start_server_computing, 0, 0.5f) { + getSyncResponse(requestParams, MAX_NUMBER_OF_RETRY_AFTER_TIMEOUT) + } + } + + if (syncResponse.isSuccessful) { + logDuration("INIT_SYNC Download and save to file") { + reportSubtask(initialSyncProgressService, R.string.initial_sync_start_downloading, 0, 0.5f) { + syncResponse.body()?.byteStream()?.use { inputStream -> + workingFile.outputStream().use { outputStream -> + inputStream.copyTo(outputStream) + } + } + } + } + } else { + throw syncResponse.toFailure(globalErrorReceiver) + .also { Timber.w("INIT_SYNC request failure: $this") } + } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADED) + } + handleSyncFile(workingFile) + + // Delete all files + workingDir.deleteRecursively() + } + + private suspend fun getSyncResponse(requestParams: Map, maxNumberOfRetries: Int): Response { + var retry = maxNumberOfRetries + while (true) { + retry-- + try { + return syncAPI.syncStream( + params = requestParams + ).awaitResponse() + } catch (throwable: Throwable) { + if (throwable is SocketTimeoutException && retry > 0) { + Timber.w("INIT_SYNC timeout retry left: $retry") + } else { + Timber.e(throwable, "INIT_SYNC timeout, no retry left, or other error") + throw throwable + } + } + } + } + + private suspend fun handleSyncFile(workingFile: File) { + val syncResponseLength = workingFile.length().toInt() + + logDuration("INIT_SYNC handleSyncFile() file size $syncResponseLength bytes") { + if (syncResponseLength < (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToSplit ?: Long.MAX_VALUE) { + // OK, no need to split just handle as a regular sync response + Timber.v("INIT_SYNC no need to split") + handleInitialSyncFile(workingFile) + } else { + Timber.v("INIT_SYNC Split into several smaller files") + // Set file mode + // TODO This is really ugly, I should improve that + LazyRoomSyncJsonAdapter.initWith(workingFile) + + handleInitialSyncFile(workingFile) + + // Reset file mode + LazyRoomSyncJsonAdapter.reset() + } + } + } + + private suspend fun handleInitialSyncFile(workingFile: File) { + val syncResponse = logDuration("INIT_SYNC Read file and parse") { + MoshiProvider.providesMoshi().adapter(SyncResponse::class.java) + .fromJson(workingFile.source().buffer())!! + } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED) + + // Log some stats + val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0 + val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored } + Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files") + + logDuration("INIT_SYNC Database insertion") { + syncResponseHandler.handleResponse(syncResponse, null) + } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS) + } + companion object { + private const val MAX_NUMBER_OF_RETRY_AFTER_TIMEOUT = 50 + private const val TIMEOUT_MARGIN: Long = 10_000 } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt new file mode 100644 index 0000000000..e7e0dcb8b9 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync.model + +import com.squareup.moshi.JsonClass +import com.squareup.moshi.JsonReader +import okio.buffer +import okio.source +import org.matrix.android.sdk.internal.di.MoshiProvider +import java.io.File + +@JsonClass(generateAdapter = false) +internal sealed class LazyRoomSync { + data class Parsed(val _roomSync: RoomSync) : LazyRoomSync() + data class Stored(val file: File) : LazyRoomSync() + + val roomSync: RoomSync + get() { + return when (this) { + is Parsed -> _roomSync + is Stored -> { + // Parse the file now + file.inputStream().use { pos -> + MoshiProvider.providesMoshi().adapter(RoomSync::class.java) + .fromJson(JsonReader.of(pos.source().buffer()))!! + } + } + } + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt new file mode 100644 index 0000000000..edf76d2e77 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync.model + +import com.squareup.moshi.FromJson +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.JsonReader +import com.squareup.moshi.JsonWriter +import com.squareup.moshi.ToJson +import org.matrix.android.sdk.internal.di.MoshiProvider +import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy +import org.matrix.android.sdk.internal.session.sync.initialSyncStrategy +import timber.log.Timber +import java.io.File +import java.util.concurrent.atomic.AtomicInteger + +internal class LazyRoomSyncJsonAdapter : JsonAdapter() { + + @FromJson + override fun fromJson(reader: JsonReader): LazyRoomSync { + return if (workingDirectory != null) { + val path = reader.path + // val roomId = reader.path.substringAfter("\$.rooms.join.") + + // inputStream.available() return 0... So read it to a String then decide to store in a file or to parse it now + val json = reader.nextSource().inputStream().bufferedReader().use { + it.readText() + } + + val limit = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToStoreInFile ?: Long.MAX_VALUE + if (json.length > limit) { + Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file") + // Copy the source to a file + val file = createFile() + file.writeText(json) + LazyRoomSync.Stored(file) + } else { + Timber.v("INIT_SYNC $path content length: ${json.length} parse it now") + // Parse it now + val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(json)!! + LazyRoomSync.Parsed(roomSync) + } + } else { + // Parse it now + val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(reader)!! + LazyRoomSync.Parsed(roomSync) + } + } + + @ToJson + override fun toJson(writer: JsonWriter, value: LazyRoomSync?) { + // This Adapter is not supposed to serialize object + throw UnsupportedOperationException() + } + + companion object { + fun initWith(file: File) { + workingDirectory = file.parentFile + atomicInteger.set(0) + } + + fun reset() { + workingDirectory = null + } + + private fun createFile(): File { + val parent = workingDirectory ?: error("workingDirectory is not initialized") + val index = atomicInteger.getAndIncrement() + + return File(parent, "room_$index.json") + } + + private var workingDirectory: File? = null + private val atomicInteger = AtomicInteger(0) + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/RoomsSyncResponse.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/RoomsSyncResponse.kt index dd2f96c988..a62d80a088 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/RoomsSyncResponse.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/RoomsSyncResponse.kt @@ -24,7 +24,7 @@ internal data class RoomsSyncResponse( /** * Joined rooms: keys are rooms ids. */ - @Json(name = "join") val join: Map = emptyMap(), + @Json(name = "join") val join: Map = emptyMap(), /** * Invitations. The rooms that the user has been invited to: keys are rooms ids. diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt new file mode 100644 index 0000000000..1a4f42a533 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.util + +import timber.log.Timber + +internal suspend fun logDuration(message: String, + block: suspend () -> T): T { + Timber.v("$message -- BEGIN") + val start = System.currentTimeMillis() + val result = block() + val duration = System.currentTimeMillis() - start + Timber.v("$message -- END duration: $duration ms") + return result +} diff --git a/matrix-sdk-android/src/main/res/values/strings.xml b/matrix-sdk-android/src/main/res/values/strings.xml index 26b9bc19d9..b93d9b680e 100644 --- a/matrix-sdk-android/src/main/res/values/strings.xml +++ b/matrix-sdk-android/src/main/res/values/strings.xml @@ -199,6 +199,8 @@ Empty room Empty room (was %s) + Initial Sync:\nWaiting for server response… + Initial Sync:\nDownloading data… Initial Sync:\nImporting account… Initial Sync:\nImporting crypto Initial Sync:\nImporting Rooms diff --git a/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt b/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt index 6a381ec049..2d884f1ba7 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt @@ -40,6 +40,8 @@ import im.vector.app.core.platform.ToolbarConfigurable import im.vector.app.core.platform.VectorBaseActivity import im.vector.app.core.pushers.PushersManager import im.vector.app.databinding.ActivityHomeBinding +import im.vector.app.features.MainActivity +import im.vector.app.features.MainActivityArgs import im.vector.app.features.disclaimer.showDisclaimerDialog import im.vector.app.features.matrixto.MatrixToBottomSheet import im.vector.app.features.notifications.NotificationDrawerManager @@ -60,6 +62,8 @@ import kotlinx.parcelize.Parcelize import org.matrix.android.sdk.api.session.InitialSyncProgressService import org.matrix.android.sdk.api.session.permalinks.PermalinkService import org.matrix.android.sdk.api.util.MatrixItem +import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy +import org.matrix.android.sdk.internal.session.sync.initialSyncStrategy import timber.log.Timber import javax.inject.Inject @@ -368,6 +372,20 @@ class HomeActivity : bugReporter.openBugReportScreen(this, false) return true } + R.id.menu_home_init_sync_legacy -> { + // Configure the SDK + initialSyncStrategy = InitialSyncStrategy.Legacy + // And clear cache + MainActivity.restartApp(this, MainActivityArgs(clearCache = true)) + return true + } + R.id.menu_home_init_sync_optimized -> { + // Configure the SDK + initialSyncStrategy = InitialSyncStrategy.Optimized() + // And clear cache + MainActivity.restartApp(this, MainActivityArgs(clearCache = true)) + return true + } R.id.menu_home_filter -> { navigator.openRoomsFiltering(this) return true diff --git a/vector/src/main/res/menu/home.xml b/vector/src/main/res/menu/home.xml index 7a77c45240..66cbf53bfd 100644 --- a/vector/src/main/res/menu/home.xml +++ b/vector/src/main/res/menu/home.xml @@ -1,6 +1,7 @@ + xmlns:app="http://schemas.android.com/apk/res-auto" + xmlns:tools="http://schemas.android.com/tools"> + + + + Date: Tue, 26 Jan 2021 13:45:17 +0100 Subject: [PATCH 02/17] Try to log RAM usage --- .../org/matrix/android/sdk/internal/util/LogUtil.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt index 1a4f42a533..2bd9f46a4c 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt @@ -20,10 +20,18 @@ import timber.log.Timber internal suspend fun logDuration(message: String, block: suspend () -> T): T { + val runtime = Runtime.getRuntime() + runtime.gc() + val usedMemInMBStart = (runtime.totalMemory() - runtime.freeMemory()) / 1048576L + Timber.v("$message -- BEGIN") val start = System.currentTimeMillis() val result = block() val duration = System.currentTimeMillis() - start - Timber.v("$message -- END duration: $duration ms") + runtime.gc() + val usedMemInMBEnd = (runtime.totalMemory() - runtime.freeMemory()) / 1048576L + val usedMemInMBDiff = usedMemInMBEnd - usedMemInMBStart + + Timber.v("$message -- END duration: $duration ms RAM usage: $usedMemInMBDiff MB") return result } From 8e2161bd9ee71823473f5e2ba97e3cd85d296c0e Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Wed, 3 Feb 2021 15:37:46 +0100 Subject: [PATCH 03/17] Log free memory before starting a task --- .../java/org/matrix/android/sdk/internal/util/LogUtil.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt index 2bd9f46a4c..4dc71655ab 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt @@ -22,9 +22,10 @@ internal suspend fun logDuration(message: String, block: suspend () -> T): T { val runtime = Runtime.getRuntime() runtime.gc() - val usedMemInMBStart = (runtime.totalMemory() - runtime.freeMemory()) / 1048576L + val freeMemoryInMb = runtime.freeMemory() / 1048576L + val usedMemInMBStart = runtime.totalMemory() / 1048576L - freeMemoryInMb - Timber.v("$message -- BEGIN") + Timber.v("$message -- BEGIN (free memory: $freeMemoryInMb MB)") val start = System.currentTimeMillis() val result = block() val duration = System.currentTimeMillis() - start From 3633199e683cfc308167d8dda4e5cc3890724d57 Mon Sep 17 00:00:00 2001 From: ganfra Date: Thu, 4 Feb 2021 14:01:21 +0100 Subject: [PATCH 04/17] Sync : clean a bit by introducing 2 moshi adapters --- matrix-sdk-android/build.gradle | 2 + .../android/sdk/internal/di/MoshiProvider.kt | 5 +- .../sdk/internal/session/sync/SyncTask.kt | 67 +++++--------- .../session/sync/model/LazyRoomSync.kt | 7 +- .../sync/model/LazyRoomSyncJsonAdapter.kt | 90 ------------------- .../sync/parsing/InitialSyncResponseParser.kt | 48 ++++++++++ .../sync/parsing/LazyRoomSyncJsonAdapters.kt | 85 ++++++++++++++++++ 7 files changed, 163 insertions(+), 141 deletions(-) delete mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index 3ade9f7214..4f8604bfc1 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -13,6 +13,7 @@ buildscript { } } + android { compileSdkVersion 30 testOptions.unitTests.includeAndroidResources = true @@ -88,6 +89,7 @@ android { java.srcDirs += "src/sharedTest/java" } } + } static def gitRevision() { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt index c0595a9273..b6bc17eaa0 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/di/MoshiProvider.kt @@ -36,7 +36,7 @@ import org.matrix.android.sdk.internal.network.parsing.ForceToBooleanJsonAdapter import org.matrix.android.sdk.internal.network.parsing.RuntimeJsonAdapterFactory import org.matrix.android.sdk.internal.network.parsing.TlsVersionMoshiAdapter import org.matrix.android.sdk.internal.network.parsing.UriMoshiAdapter -import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter +import org.matrix.android.sdk.internal.session.sync.parsing.DefaultLazyRoomSyncJsonAdapter object MoshiProvider { @@ -45,7 +45,8 @@ object MoshiProvider { .add(ForceToBooleanJsonAdapter()) .add(CipherSuiteMoshiAdapter()) .add(TlsVersionMoshiAdapter()) - .add(LazyRoomSyncJsonAdapter()) + // Use addLast here so we can inject a SplitLazyRoomSyncJsonAdapter later to override the default parsing. + .addLast(DefaultLazyRoomSyncJsonAdapter()) .add(RuntimeJsonAdapterFactory.of(MessageContent::class.java, "msgtype", MessageDefaultContent::class.java) .registerSubtype(MessageTextContent::class.java, MessageType.MSGTYPE_TEXT) .registerSubtype(MessageNoticeContent::class.java, MessageType.MSGTYPE_NOTICE) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt index ef766f6d42..dde3da48c6 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt @@ -17,10 +17,7 @@ package org.matrix.android.sdk.internal.session.sync import okhttp3.ResponseBody -import okio.buffer -import okio.source import org.matrix.android.sdk.R -import org.matrix.android.sdk.internal.di.MoshiProvider import org.matrix.android.sdk.internal.di.SessionFilesDirectory import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.network.GlobalErrorReceiver @@ -32,8 +29,8 @@ import org.matrix.android.sdk.internal.session.filter.FilterRepository import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask import org.matrix.android.sdk.internal.session.reportSubtask import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync -import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSyncJsonAdapter import org.matrix.android.sdk.internal.session.sync.model.SyncResponse +import org.matrix.android.sdk.internal.session.sync.parsing.InitialSyncResponseParser import org.matrix.android.sdk.internal.session.user.UserStore import org.matrix.android.sdk.internal.task.Task import org.matrix.android.sdk.internal.util.logDuration @@ -64,7 +61,8 @@ internal class DefaultSyncTask @Inject constructor( private val syncTaskSequencer: SyncTaskSequencer, private val globalErrorReceiver: GlobalErrorReceiver, @SessionFilesDirectory - private val fileDirectory: File + private val fileDirectory: File, + private val syncResponseParser: InitialSyncResponseParser ) : SyncTask { private val workingDir = File(fileDirectory, "is") @@ -101,9 +99,10 @@ internal class DefaultSyncTask @Inject constructor( val readTimeOut = (params.timeout + TIMEOUT_MARGIN).coerceAtLeast(TimeOutInterceptor.DEFAULT_LONG_TIMEOUT) if (isInitialSync) { - logDuration("INIT_SYNC strategy: $initialSyncStrategy") { - if (initialSyncStrategy is InitialSyncStrategy.Optimized) { - safeInitialSync(requestParams) + val initSyncStrategy = initialSyncStrategy + logDuration("INIT_SYNC strategy: $initSyncStrategy") { + if (initSyncStrategy is InitialSyncStrategy.Optimized) { + safeInitialSync(requestParams, initSyncStrategy) } else { val syncResponse = logDuration("INIT_SYNC Request") { executeRequest(globalErrorReceiver) { @@ -132,7 +131,7 @@ internal class DefaultSyncTask @Inject constructor( Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}") } - private suspend fun safeInitialSync(requestParams: Map) { + private suspend fun safeInitialSync(requestParams: Map, initSyncStrategy: InitialSyncStrategy.Optimized) { workingDir.mkdirs() val workingFile = File(workingDir, "initSync.json") val status = initialSyncStatusRepository.getStep() @@ -163,7 +162,7 @@ internal class DefaultSyncTask @Inject constructor( } initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADED) } - handleSyncFile(workingFile) + handleSyncFile(workingFile, initSyncStrategy) // Delete all files workingDir.deleteRecursively() @@ -188,44 +187,22 @@ internal class DefaultSyncTask @Inject constructor( } } - private suspend fun handleSyncFile(workingFile: File) { - val syncResponseLength = workingFile.length().toInt() - - logDuration("INIT_SYNC handleSyncFile() file size $syncResponseLength bytes") { - if (syncResponseLength < (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToSplit ?: Long.MAX_VALUE) { - // OK, no need to split just handle as a regular sync response - Timber.v("INIT_SYNC no need to split") - handleInitialSyncFile(workingFile) - } else { - Timber.v("INIT_SYNC Split into several smaller files") - // Set file mode - // TODO This is really ugly, I should improve that - LazyRoomSyncJsonAdapter.initWith(workingFile) - - handleInitialSyncFile(workingFile) - - // Reset file mode - LazyRoomSyncJsonAdapter.reset() + private suspend fun handleSyncFile(workingFile: File, initSyncStrategy: InitialSyncStrategy.Optimized) { + logDuration("INIT_SYNC handleSyncFile()") { + val syncResponse = logDuration("INIT_SYNC Read file and parse") { + syncResponseParser.parse(initSyncStrategy, workingFile) } - } - } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED) + // Log some stats + val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0 + val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored } + Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files") - private suspend fun handleInitialSyncFile(workingFile: File) { - val syncResponse = logDuration("INIT_SYNC Read file and parse") { - MoshiProvider.providesMoshi().adapter(SyncResponse::class.java) - .fromJson(workingFile.source().buffer())!! + logDuration("INIT_SYNC Database insertion") { + syncResponseHandler.handleResponse(syncResponse, null) + } + initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS) } - initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_PARSED) - - // Log some stats - val nbOfJoinedRooms = syncResponse.rooms?.join?.size ?: 0 - val nbOfJoinedRoomsInFile = syncResponse.rooms?.join?.values?.count { it is LazyRoomSync.Stored } - Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files") - - logDuration("INIT_SYNC Database insertion") { - syncResponseHandler.handleResponse(syncResponse, null) - } - initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS) } companion object { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt index e7e0dcb8b9..1f6e3de9a4 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSync.kt @@ -16,17 +16,17 @@ package org.matrix.android.sdk.internal.session.sync.model +import com.squareup.moshi.JsonAdapter import com.squareup.moshi.JsonClass import com.squareup.moshi.JsonReader import okio.buffer import okio.source -import org.matrix.android.sdk.internal.di.MoshiProvider import java.io.File @JsonClass(generateAdapter = false) internal sealed class LazyRoomSync { data class Parsed(val _roomSync: RoomSync) : LazyRoomSync() - data class Stored(val file: File) : LazyRoomSync() + data class Stored(val roomSyncAdapter: JsonAdapter, val file: File) : LazyRoomSync() val roomSync: RoomSync get() { @@ -35,8 +35,7 @@ internal sealed class LazyRoomSync { is Stored -> { // Parse the file now file.inputStream().use { pos -> - MoshiProvider.providesMoshi().adapter(RoomSync::class.java) - .fromJson(JsonReader.of(pos.source().buffer()))!! + roomSyncAdapter.fromJson(JsonReader.of(pos.source().buffer()))!! } } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt deleted file mode 100644 index edf76d2e77..0000000000 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/model/LazyRoomSyncJsonAdapter.kt +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2021 The Matrix.org Foundation C.I.C. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.matrix.android.sdk.internal.session.sync.model - -import com.squareup.moshi.FromJson -import com.squareup.moshi.JsonAdapter -import com.squareup.moshi.JsonReader -import com.squareup.moshi.JsonWriter -import com.squareup.moshi.ToJson -import org.matrix.android.sdk.internal.di.MoshiProvider -import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy -import org.matrix.android.sdk.internal.session.sync.initialSyncStrategy -import timber.log.Timber -import java.io.File -import java.util.concurrent.atomic.AtomicInteger - -internal class LazyRoomSyncJsonAdapter : JsonAdapter() { - - @FromJson - override fun fromJson(reader: JsonReader): LazyRoomSync { - return if (workingDirectory != null) { - val path = reader.path - // val roomId = reader.path.substringAfter("\$.rooms.join.") - - // inputStream.available() return 0... So read it to a String then decide to store in a file or to parse it now - val json = reader.nextSource().inputStream().bufferedReader().use { - it.readText() - } - - val limit = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.minSizeToStoreInFile ?: Long.MAX_VALUE - if (json.length > limit) { - Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file") - // Copy the source to a file - val file = createFile() - file.writeText(json) - LazyRoomSync.Stored(file) - } else { - Timber.v("INIT_SYNC $path content length: ${json.length} parse it now") - // Parse it now - val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(json)!! - LazyRoomSync.Parsed(roomSync) - } - } else { - // Parse it now - val roomSync = MoshiProvider.providesMoshi().adapter(RoomSync::class.java).fromJson(reader)!! - LazyRoomSync.Parsed(roomSync) - } - } - - @ToJson - override fun toJson(writer: JsonWriter, value: LazyRoomSync?) { - // This Adapter is not supposed to serialize object - throw UnsupportedOperationException() - } - - companion object { - fun initWith(file: File) { - workingDirectory = file.parentFile - atomicInteger.set(0) - } - - fun reset() { - workingDirectory = null - } - - private fun createFile(): File { - val parent = workingDirectory ?: error("workingDirectory is not initialized") - val index = atomicInteger.getAndIncrement() - - return File(parent, "room_$index.json") - } - - private var workingDirectory: File? = null - private val atomicInteger = AtomicInteger(0) - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt new file mode 100644 index 0000000000..760f25270f --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync.parsing + +import com.squareup.moshi.Moshi +import okio.buffer +import okio.source +import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy +import org.matrix.android.sdk.internal.session.sync.model.SyncResponse +import timber.log.Timber +import java.io.File +import javax.inject.Inject + +internal class InitialSyncResponseParser @Inject constructor(private val moshi: Moshi) { + + fun parse(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File): SyncResponse { + val syncResponseLength = workingFile.length().toInt() + Timber.v("Sync file size is $syncResponseLength bytes") + val shouldSplit = syncResponseLength >= syncStrategy.minSizeToSplit + Timber.v("INIT_SYNC should split in several files: $shouldSplit") + return getMoshi(syncStrategy, workingFile, shouldSplit) + .adapter(SyncResponse::class.java) + .fromJson(workingFile.source().buffer())!! + } + + private fun getMoshi(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File, shouldSplit: Boolean): Moshi { + // If we don't have to split we'll rely on the already default moshi + if (!shouldSplit) return moshi + // Otherwise, we create a new adapter for handling Map of Lazy sync + return moshi.newBuilder() + .add(SplitLazyRoomSyncJsonAdapter(workingFile, syncStrategy)) + .build() + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt new file mode 100644 index 0000000000..232e2fe99e --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2021 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.sync.parsing + +import com.squareup.moshi.FromJson +import com.squareup.moshi.JsonAdapter +import com.squareup.moshi.JsonReader +import com.squareup.moshi.JsonWriter +import com.squareup.moshi.ToJson +import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy +import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync +import org.matrix.android.sdk.internal.session.sync.model.RoomSync +import timber.log.Timber +import java.io.File +import java.util.concurrent.atomic.AtomicInteger + +internal class DefaultLazyRoomSyncJsonAdapter() { + + @FromJson + fun fromJson(reader: JsonReader, delegate: JsonAdapter): LazyRoomSync? { + val roomSync = delegate.fromJson(reader) ?: return null + return LazyRoomSync.Parsed(roomSync) + } + + @ToJson + fun toJson(writer: JsonWriter, value: LazyRoomSync?) { + // This Adapter is not supposed to serialize object + Timber.v("To json $value with $writer") + throw UnsupportedOperationException() + } +} + +internal class SplitLazyRoomSyncJsonAdapter( + private val workingDirectory: File, + private val syncStrategy: InitialSyncStrategy.Optimized +) { + + private val atomicInteger = AtomicInteger(0) + + private fun createFile(): File { + val index = atomicInteger.getAndIncrement() + return File(workingDirectory.parentFile, "room_$index.json") + } + + @FromJson + fun fromJson(reader: JsonReader, delegate: JsonAdapter): LazyRoomSync? { + val path = reader.path + val json = reader.nextSource().inputStream().bufferedReader().use { + it.readText() + } + val limit = syncStrategy.minSizeToStoreInFile + return if (json.length > limit) { + Timber.v("INIT_SYNC $path content length: ${json.length} copy to a file") + // Copy the source to a file + val file = createFile() + file.writeText(json) + LazyRoomSync.Stored(delegate, file) + } else { + Timber.v("INIT_SYNC $path content length: ${json.length} parse it now") + val roomSync = delegate.fromJson(json) ?: return null + LazyRoomSync.Parsed(roomSync) + } + } + + @ToJson + fun toJson(writer: JsonWriter, value: LazyRoomSync?) { + // This Adapter is not supposed to serialize object + Timber.v("To json $value with $writer") + throw UnsupportedOperationException() + } +} From 218be86c683cadc6c864214df9287367da881c87 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 8 Feb 2021 18:14:57 +0100 Subject: [PATCH 05/17] Cleanup --- matrix-sdk-android/build.gradle | 2 -- .../session/sync/parsing/InitialSyncResponseParser.kt | 2 +- .../session/sync/parsing/LazyRoomSyncJsonAdapters.kt | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index 4f8604bfc1..3ade9f7214 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -13,7 +13,6 @@ buildscript { } } - android { compileSdkVersion 30 testOptions.unitTests.includeAndroidResources = true @@ -89,7 +88,6 @@ android { java.srcDirs += "src/sharedTest/java" } } - } static def gitRevision() { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt index 760f25270f..8368351419 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 New Vector Ltd + * Copyright (c) 2021 The Matrix.org Foundation C.I.C. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt index 232e2fe99e..2d25d0bc2e 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 New Vector Ltd + * Copyright (c) 2021 The Matrix.org Foundation C.I.C. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ import timber.log.Timber import java.io.File import java.util.concurrent.atomic.AtomicInteger -internal class DefaultLazyRoomSyncJsonAdapter() { +internal class DefaultLazyRoomSyncJsonAdapter { @FromJson fun fromJson(reader: JsonReader, delegate: JsonAdapter): LazyRoomSync? { @@ -48,7 +48,6 @@ internal class SplitLazyRoomSyncJsonAdapter( private val workingDirectory: File, private val syncStrategy: InitialSyncStrategy.Optimized ) { - private val atomicInteger = AtomicInteger(0) private fun createFile(): File { From 440d01c5525c3848b7a4cb78daf45d6ffad3e449 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 8 Feb 2021 18:29:22 +0100 Subject: [PATCH 06/17] Do not log RAM usage in release --- .../android/sdk/internal/util/LogUtil.kt | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt index 4dc71655ab..fe68b49a5c 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/util/LogUtil.kt @@ -16,23 +16,36 @@ package org.matrix.android.sdk.internal.util +import org.matrix.android.sdk.BuildConfig import timber.log.Timber internal suspend fun logDuration(message: String, block: suspend () -> T): T { - val runtime = Runtime.getRuntime() - runtime.gc() - val freeMemoryInMb = runtime.freeMemory() / 1048576L - val usedMemInMBStart = runtime.totalMemory() / 1048576L - freeMemoryInMb - - Timber.v("$message -- BEGIN (free memory: $freeMemoryInMb MB)") + Timber.v("$message -- BEGIN") val start = System.currentTimeMillis() - val result = block() + val result = logRamUsage(message) { + block() + } val duration = System.currentTimeMillis() - start - runtime.gc() - val usedMemInMBEnd = (runtime.totalMemory() - runtime.freeMemory()) / 1048576L - val usedMemInMBDiff = usedMemInMBEnd - usedMemInMBStart + Timber.v("$message -- END duration: $duration ms") - Timber.v("$message -- END duration: $duration ms RAM usage: $usedMemInMBDiff MB") return result } + +internal suspend fun logRamUsage(message: String, block: suspend () -> T): T { + return if (BuildConfig.DEBUG) { + val runtime = Runtime.getRuntime() + runtime.gc() + val freeMemoryInMb = runtime.freeMemory() / 1048576L + val usedMemInMBStart = runtime.totalMemory() / 1048576L - freeMemoryInMb + Timber.v("$message -- BEGIN (free memory: $freeMemoryInMb MB)") + val result = block() + runtime.gc() + val usedMemInMBEnd = (runtime.totalMemory() - runtime.freeMemory()) / 1048576L + val usedMemInMBDiff = usedMemInMBEnd - usedMemInMBStart + Timber.v("$message -- END RAM usage: $usedMemInMBDiff MB") + result + } else { + block() + } +} From e069e196af71e4c3d0a737f9cf3add080f699f5d Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 8 Feb 2021 18:42:49 +0100 Subject: [PATCH 07/17] Improve log --- .../internal/session/sync/parsing/InitialSyncResponseParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt index 8368351419..6145793178 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt @@ -29,7 +29,7 @@ internal class InitialSyncResponseParser @Inject constructor(private val moshi: fun parse(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File): SyncResponse { val syncResponseLength = workingFile.length().toInt() - Timber.v("Sync file size is $syncResponseLength bytes") + Timber.v("INIT_SYNC Sync file size is $syncResponseLength bytes") val shouldSplit = syncResponseLength >= syncStrategy.minSizeToSplit Timber.v("INIT_SYNC should split in several files: $shouldSplit") return getMoshi(syncStrategy, workingFile, shouldSplit) From a2225b3f760ae56702719ec44a48f1e39de3db23 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 8 Feb 2021 18:56:41 +0100 Subject: [PATCH 08/17] Improve code about path --- .../session/sync/parsing/InitialSyncResponseParser.kt | 6 +++--- .../session/sync/parsing/LazyRoomSyncJsonAdapters.kt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt index 6145793178..ae7b2a4468 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/InitialSyncResponseParser.kt @@ -32,17 +32,17 @@ internal class InitialSyncResponseParser @Inject constructor(private val moshi: Timber.v("INIT_SYNC Sync file size is $syncResponseLength bytes") val shouldSplit = syncResponseLength >= syncStrategy.minSizeToSplit Timber.v("INIT_SYNC should split in several files: $shouldSplit") - return getMoshi(syncStrategy, workingFile, shouldSplit) + return getMoshi(syncStrategy, workingFile.parentFile!!, shouldSplit) .adapter(SyncResponse::class.java) .fromJson(workingFile.source().buffer())!! } - private fun getMoshi(syncStrategy: InitialSyncStrategy.Optimized, workingFile: File, shouldSplit: Boolean): Moshi { + private fun getMoshi(syncStrategy: InitialSyncStrategy.Optimized, workingDirectory: File, shouldSplit: Boolean): Moshi { // If we don't have to split we'll rely on the already default moshi if (!shouldSplit) return moshi // Otherwise, we create a new adapter for handling Map of Lazy sync return moshi.newBuilder() - .add(SplitLazyRoomSyncJsonAdapter(workingFile, syncStrategy)) + .add(SplitLazyRoomSyncJsonAdapter(workingDirectory, syncStrategy)) .build() } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt index 2d25d0bc2e..951fdf86aa 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/parsing/LazyRoomSyncJsonAdapters.kt @@ -52,7 +52,7 @@ internal class SplitLazyRoomSyncJsonAdapter( private fun createFile(): File { val index = atomicInteger.getAndIncrement() - return File(workingDirectory.parentFile, "room_$index.json") + return File(workingDirectory, "room_$index.json") } @FromJson From 4e1fcf87ae2c4e4cd276fd24875642f1809554ea Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 8 Feb 2021 19:44:02 +0100 Subject: [PATCH 09/17] Improve DefaultInitialSyncProgressService --- .../DefaultInitialSyncProgressService.kt | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt index 9918e83fbc..9c816964db 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt @@ -19,11 +19,14 @@ import androidx.annotation.StringRes import androidx.lifecycle.LiveData import androidx.lifecycle.MutableLiveData import org.matrix.android.sdk.api.session.InitialSyncProgressService +import org.matrix.android.sdk.internal.util.StringProvider import timber.log.Timber import javax.inject.Inject @SessionScope -class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgressService { +internal class DefaultInitialSyncProgressService @Inject constructor( + private val stringProvider: StringProvider +) : InitialSyncProgressService { private val status = MutableLiveData() @@ -40,10 +43,12 @@ class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgr } else { val currentLeaf = rootTask!!.leaf() - val newTask = TaskInfo(nameRes, - totalProgress, - currentLeaf, - parentWeight) + val newTask = TaskInfo( + nameRes = nameRes, + totalProgress = totalProgress, + parent = currentLeaf, + parentWeight = parentWeight + ) currentLeaf.child = newTask } @@ -72,11 +77,11 @@ class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgr status.postValue(InitialSyncProgressService.Status.Idle) } - private inner class TaskInfo(@StringRes var nameRes: Int, - var totalProgress: Int, - var parent: TaskInfo? = null, - var parentWeight: Float = 1f, - var offset: Int = parent?.currentProgress ?: 0) { + private inner class TaskInfo(@StringRes val nameRes: Int, + val totalProgress: Int, + val parent: TaskInfo? = null, + val parentWeight: Float = 1f, + val offset: Int = parent?.currentProgress ?: 0) { var child: TaskInfo? = null var currentProgress: Int = 0 @@ -97,32 +102,32 @@ class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgr fun setProgress(progress: Int) { currentProgress = progress // val newProgress = Math.min(currentProgress + progress, totalProgress) - parent?.let { + if (parent != null) { val parentProgress = (currentProgress * parentWeight).toInt() - it.setProgress(offset + parentProgress) - } ?: run { - Timber.v("--- ${leaf().nameRes}: $currentProgress") + parent.setProgress(offset + parentProgress) + } else { + Timber.v("--- ${stringProvider.getString(leaf().nameRes)}: $currentProgress") status.postValue(InitialSyncProgressService.Status.Progressing(leaf().nameRes, currentProgress)) } } } } -inline fun reportSubtask(reporter: DefaultInitialSyncProgressService?, - @StringRes nameRes: Int, - totalProgress: Int, - parentWeight: Float = 1f, - block: () -> T): T { +internal inline fun reportSubtask(reporter: DefaultInitialSyncProgressService?, + @StringRes nameRes: Int, + totalProgress: Int, + parentWeight: Float = 1f, + block: () -> T): T { reporter?.startTask(nameRes, totalProgress, parentWeight) return block().also { reporter?.endTask(nameRes) } } -inline fun Map.mapWithProgress(reporter: DefaultInitialSyncProgressService?, - taskId: Int, - weight: Float, - transform: (Map.Entry) -> R): List { +internal inline fun Map.mapWithProgress(reporter: DefaultInitialSyncProgressService?, + taskId: Int, + weight: Float, + transform: (Map.Entry) -> R): List { val total = count().toFloat() var current = 0 reporter?.startTask(taskId, 100, weight) From 5e7b36d319d54920610af47d6b49871376754c40 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Mon, 8 Feb 2021 22:49:07 +0100 Subject: [PATCH 10/17] Fix issue with progress --- .../DefaultInitialSyncProgressService.kt | 40 +++++++++---------- .../session/sync/CryptoSyncHandler.kt | 2 +- .../internal/session/sync/RoomSyncHandler.kt | 2 +- .../session/sync/SyncResponseHandler.kt | 12 +++--- .../sdk/internal/session/sync/SyncTask.kt | 19 +++++---- 5 files changed, 39 insertions(+), 36 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt index 9c816964db..d84a39f462 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt @@ -36,7 +36,7 @@ internal class DefaultInitialSyncProgressService @Inject constructor( return status } - fun startTask(@StringRes nameRes: Int, totalProgress: Int, parentWeight: Float = 1f) { + fun startTask(@StringRes nameRes: Int, totalProgress: Int, parentWeight: Float) { // Create a rootTask, or add a child to the leaf if (rootTask == null) { rootTask = TaskInfo(nameRes, totalProgress) @@ -52,20 +52,20 @@ internal class DefaultInitialSyncProgressService @Inject constructor( currentLeaf.child = newTask } - reportProgress(0) + reportProgress(0F) } - fun reportProgress(progress: Int) { + fun reportProgress(progress: Float) { rootTask?.leaf()?.setProgress(progress) } fun endTask(nameRes: Int) { val endedTask = rootTask?.leaf() if (endedTask?.nameRes == nameRes) { - // close it - val parent = endedTask.parent - parent?.child = null - parent?.setProgress(endedTask.offset + (endedTask.totalProgress * endedTask.parentWeight).toInt()) + // Ensure the task progress is complete + endedTask.setProgress(endedTask.totalProgress.toFloat()) + // And close it + endedTask.parent?.child = null } if (endedTask?.parent == null) { status.postValue(InitialSyncProgressService.Status.Idle) @@ -81,9 +81,9 @@ internal class DefaultInitialSyncProgressService @Inject constructor( val totalProgress: Int, val parent: TaskInfo? = null, val parentWeight: Float = 1f, - val offset: Int = parent?.currentProgress ?: 0) { + val offset: Float = parent?.currentProgress ?: 0F) { var child: TaskInfo? = null - var currentProgress: Int = 0 + private var currentProgress: Float = 0F /** * Get the further child @@ -97,17 +97,18 @@ internal class DefaultInitialSyncProgressService @Inject constructor( } /** - * Set progress of the parent if any (which will post value), or post the value + * Set progress of this task and update the parent progress. Last parent will post value. */ - fun setProgress(progress: Int) { + fun setProgress(progress: Float) { + Timber.v("setProgress: $progress / $totalProgress") currentProgress = progress // val newProgress = Math.min(currentProgress + progress, totalProgress) if (parent != null) { - val parentProgress = (currentProgress * parentWeight).toInt() + val parentProgress = (currentProgress / totalProgress) * (parentWeight * parent.totalProgress) parent.setProgress(offset + parentProgress) } else { - Timber.v("--- ${stringProvider.getString(leaf().nameRes)}: $currentProgress") - status.postValue(InitialSyncProgressService.Status.Progressing(leaf().nameRes, currentProgress)) + Timber.v("--- ${stringProvider.getString(leaf().nameRes)}: ${currentProgress.toInt()}") + status.postValue(InitialSyncProgressService.Status.Progressing(leaf().nameRes, currentProgress.toInt())) } } } @@ -116,7 +117,7 @@ internal class DefaultInitialSyncProgressService @Inject constructor( internal inline fun reportSubtask(reporter: DefaultInitialSyncProgressService?, @StringRes nameRes: Int, totalProgress: Int, - parentWeight: Float = 1f, + parentWeight: Float, block: () -> T): T { reporter?.startTask(nameRes, totalProgress, parentWeight) return block().also { @@ -126,13 +127,12 @@ internal inline fun reportSubtask(reporter: DefaultInitialSyncProgressServic internal inline fun Map.mapWithProgress(reporter: DefaultInitialSyncProgressService?, taskId: Int, - weight: Float, + parentWeight: Float, transform: (Map.Entry) -> R): List { - val total = count().toFloat() - var current = 0 - reporter?.startTask(taskId, 100, weight) + var current = 0F + reporter?.startTask(taskId, count() + 1, parentWeight) return map { - reporter?.reportProgress((current / total * 100).toInt()) + reporter?.reportProgress(current) current++ transform.invoke(it) }.also { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt index fc476a3dd6..42477e1dd8 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt @@ -38,7 +38,7 @@ internal class CryptoSyncHandler @Inject constructor(private val cryptoService: fun handleToDevice(toDevice: ToDeviceSyncResponse, initialSyncProgressService: DefaultInitialSyncProgressService? = null) { val total = toDevice.events?.size ?: 0 toDevice.events?.forEachIndexed { index, event -> - initialSyncProgressService?.reportProgress(((index / total.toFloat()) * 100).toInt()) + initialSyncProgressService?.reportProgress(index * 100F / total) // Decrypt event if necessary Timber.i("## CRYPTO | To device event from ${event.senderId} of type:${event.type}") decryptToDeviceEvent(event, null) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt index 979c2888d3..ef2212cdab 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt @@ -162,7 +162,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle ) } realm.insertOrUpdate(roomEntities) - reporter?.reportProgress(index + 1) + reporter?.reportProgress(index + 1F) } } } else { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt index a80b062427..c9376eb9f2 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt @@ -51,13 +51,11 @@ internal class SyncResponseHandler @Inject constructor(@SessionDatabase private private val cryptoService: DefaultCryptoService, private val tokenStore: SyncTokenStore, private val processEventForPushTask: ProcessEventForPushTask, - private val pushRuleService: PushRuleService, - private val initialSyncProgressService: DefaultInitialSyncProgressService) { + private val pushRuleService: PushRuleService) { - suspend fun handleResponse(syncResponse: SyncResponse, fromToken: String?) { + suspend fun handleResponse(syncResponse: SyncResponse, fromToken: String?, reporter: DefaultInitialSyncProgressService?) { val isInitialSync = fromToken == null Timber.v("Start handling sync, is InitialSync: $isInitialSync") - val reporter = initialSyncProgressService.takeIf { isInitialSync } measureTimeMillis { if (!cryptoService.isStarted()) { @@ -85,7 +83,7 @@ internal class SyncResponseHandler @Inject constructor(@SessionDatabase private monarchy.awaitTransaction { realm -> measureTimeMillis { Timber.v("Handle rooms") - reportSubtask(reporter, R.string.initial_sync_start_importing_account_rooms, 100, 0.7f) { + reportSubtask(reporter, R.string.initial_sync_start_importing_account_rooms, 1, 0.7f) { if (syncResponse.rooms != null) { roomSyncHandler.handle(realm, syncResponse.rooms, isInitialSync, reporter) } @@ -95,7 +93,7 @@ internal class SyncResponseHandler @Inject constructor(@SessionDatabase private } measureTimeMillis { - reportSubtask(reporter, R.string.initial_sync_start_importing_account_groups, 100, 0.1f) { + reportSubtask(reporter, R.string.initial_sync_start_importing_account_groups, 1, 0.1f) { Timber.v("Handle groups") if (syncResponse.groups != null) { groupSyncHandler.handle(realm, syncResponse.groups, reporter) @@ -106,7 +104,7 @@ internal class SyncResponseHandler @Inject constructor(@SessionDatabase private } measureTimeMillis { - reportSubtask(reporter, R.string.initial_sync_start_importing_account_data, 100, 0.1f) { + reportSubtask(reporter, R.string.initial_sync_start_importing_account_data, 1, 0.1f) { Timber.v("Handle accountData") userAccountDataSyncHandler.handle(realm, syncResponse.accountData) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt index dde3da48c6..ea06db7071 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt @@ -91,7 +91,7 @@ internal class DefaultSyncTask @Inject constructor( // We might want to get the user information in parallel too userStore.createOrUpdate(userId) initialSyncProgressService.endAll() - initialSyncProgressService.startTask(R.string.initial_sync_start_importing_account, 100) + initialSyncProgressService.startTask(R.string.initial_sync_start_importing_account, 100, 1F) } // Maybe refresh the home server capabilities data we know getHomeServerCapabilitiesTask.execute(GetHomeServerCapabilitiesTask.Params(forceRefresh = false)) @@ -114,7 +114,7 @@ internal class DefaultSyncTask @Inject constructor( } logDuration("INIT_SYNC Database insertion") { - syncResponseHandler.handleResponse(syncResponse, token) + syncResponseHandler.handleResponse(syncResponse, token, initialSyncProgressService) } } } @@ -126,7 +126,7 @@ internal class DefaultSyncTask @Inject constructor( readTimeOut = readTimeOut ) } - syncResponseHandler.handleResponse(syncResponse, token) + syncResponseHandler.handleResponse(syncResponse, token, null) } Timber.v("Sync task finished on Thread: ${Thread.currentThread().name}") } @@ -138,17 +138,20 @@ internal class DefaultSyncTask @Inject constructor( if (workingFile.exists() && status >= InitialSyncStatus.STEP_DOWNLOADED) { // Go directly to the parse step Timber.v("INIT_SYNC file is already here") + reportSubtask(initialSyncProgressService, R.string.initial_sync_start_downloading, 1, 0.3f) { + // Empty task + } } else { initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADING) val syncResponse = logDuration("INIT_SYNC Perform server request") { - reportSubtask(initialSyncProgressService, R.string.initial_sync_start_server_computing, 0, 0.5f) { + reportSubtask(initialSyncProgressService, R.string.initial_sync_start_server_computing, 1, 0.2f) { getSyncResponse(requestParams, MAX_NUMBER_OF_RETRY_AFTER_TIMEOUT) } } if (syncResponse.isSuccessful) { logDuration("INIT_SYNC Download and save to file") { - reportSubtask(initialSyncProgressService, R.string.initial_sync_start_downloading, 0, 0.5f) { + reportSubtask(initialSyncProgressService, R.string.initial_sync_start_downloading, 1, 0.1f) { syncResponse.body()?.byteStream()?.use { inputStream -> workingFile.outputStream().use { outputStream -> inputStream.copyTo(outputStream) @@ -162,7 +165,9 @@ internal class DefaultSyncTask @Inject constructor( } initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_DOWNLOADED) } - handleSyncFile(workingFile, initSyncStrategy) + reportSubtask(initialSyncProgressService, R.string.initial_sync_start_importing_account, 1, 0.7F) { + handleSyncFile(workingFile, initSyncStrategy) + } // Delete all files workingDir.deleteRecursively() @@ -199,7 +204,7 @@ internal class DefaultSyncTask @Inject constructor( Timber.v("INIT_SYNC $nbOfJoinedRooms rooms, $nbOfJoinedRoomsInFile stored into files") logDuration("INIT_SYNC Database insertion") { - syncResponseHandler.handleResponse(syncResponse, null) + syncResponseHandler.handleResponse(syncResponse, null, initialSyncProgressService) } initialSyncStatusRepository.setStep(InitialSyncStatus.STEP_SUCCESS) } From 1c83ee086a2db25acb3364f01733513611d0036f Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Tue, 9 Feb 2021 09:55:28 +0100 Subject: [PATCH 11/17] More rework of DefaultInitialSyncProgressService --- .../DefaultInitialSyncProgressService.kt | 109 +++++++++--------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt index d84a39f462..fb7ba7c58f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt @@ -19,14 +19,11 @@ import androidx.annotation.StringRes import androidx.lifecycle.LiveData import androidx.lifecycle.MutableLiveData import org.matrix.android.sdk.api.session.InitialSyncProgressService -import org.matrix.android.sdk.internal.util.StringProvider import timber.log.Timber import javax.inject.Inject @SessionScope -internal class DefaultInitialSyncProgressService @Inject constructor( - private val stringProvider: StringProvider -) : InitialSyncProgressService { +internal class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgressService { private val status = MutableLiveData() @@ -36,39 +33,47 @@ internal class DefaultInitialSyncProgressService @Inject constructor( return status } - fun startTask(@StringRes nameRes: Int, totalProgress: Int, parentWeight: Float) { + fun startTask(@StringRes nameRes: Int, + totalProgress: Int, + parentWeight: Float) { // Create a rootTask, or add a child to the leaf if (rootTask == null) { - rootTask = TaskInfo(nameRes, totalProgress) + rootTask = TaskInfo(nameRes, totalProgress, null, 1F) } else { val currentLeaf = rootTask!!.leaf() - - val newTask = TaskInfo( + currentLeaf.child = TaskInfo( nameRes = nameRes, totalProgress = totalProgress, parent = currentLeaf, parentWeight = parentWeight ) - - currentLeaf.child = newTask } reportProgress(0F) } fun reportProgress(progress: Float) { - rootTask?.leaf()?.setProgress(progress) + rootTask?.let { root -> + root.leaf().let { leaf -> + // Update the progress of the leaf and all its parents + leaf.setProgress(progress) + // Then update the live data using leaf wording and root progress + status.postValue(InitialSyncProgressService.Status.Progressing(leaf.nameRes, root.currentProgress.toInt())) + } + } } - fun endTask(nameRes: Int) { - val endedTask = rootTask?.leaf() - if (endedTask?.nameRes == nameRes) { + fun endTask() { + rootTask?.leaf()?.let { endedTask -> // Ensure the task progress is complete - endedTask.setProgress(endedTask.totalProgress.toFloat()) - // And close it + reportProgress(endedTask.totalProgress.toFloat()) endedTask.parent?.child = null - } - if (endedTask?.parent == null) { - status.postValue(InitialSyncProgressService.Status.Idle) + + if (endedTask.parent != null) { + // And close it + endedTask.parent.child = null + } else { + status.postValue(InitialSyncProgressService.Status.Idle) + } } } @@ -76,40 +81,38 @@ internal class DefaultInitialSyncProgressService @Inject constructor( rootTask = null status.postValue(InitialSyncProgressService.Status.Idle) } +} - private inner class TaskInfo(@StringRes val nameRes: Int, - val totalProgress: Int, - val parent: TaskInfo? = null, - val parentWeight: Float = 1f, - val offset: Float = parent?.currentProgress ?: 0F) { - var child: TaskInfo? = null - private var currentProgress: Float = 0F +private class TaskInfo(@StringRes val nameRes: Int, + val totalProgress: Int, + val parent: TaskInfo?, + val parentWeight: Float) { + var child: TaskInfo? = null + var currentProgress = 0F + private set + private val offset = parent?.currentProgress ?: 0F - /** - * Get the further child - */ - fun leaf(): TaskInfo { - var last = this - while (last.child != null) { - last = last.child!! - } - return last + /** + * Get the further child + */ + fun leaf(): TaskInfo { + var last = this + while (last.child != null) { + last = last.child!! } + return last + } - /** - * Set progress of this task and update the parent progress. Last parent will post value. - */ - fun setProgress(progress: Float) { - Timber.v("setProgress: $progress / $totalProgress") - currentProgress = progress -// val newProgress = Math.min(currentProgress + progress, totalProgress) - if (parent != null) { - val parentProgress = (currentProgress / totalProgress) * (parentWeight * parent.totalProgress) - parent.setProgress(offset + parentProgress) - } else { - Timber.v("--- ${stringProvider.getString(leaf().nameRes)}: ${currentProgress.toInt()}") - status.postValue(InitialSyncProgressService.Status.Progressing(leaf().nameRes, currentProgress.toInt())) - } + /** + * Set progress of this task and update the parent progress iteratively + */ + fun setProgress(progress: Float) { + Timber.v("setProgress: $progress / $totalProgress") + currentProgress = progress + + parent?.let { + val parentProgress = (currentProgress / totalProgress) * (parentWeight * it.totalProgress) + it.setProgress(offset + parentProgress) } } } @@ -121,21 +124,21 @@ internal inline fun reportSubtask(reporter: DefaultInitialSyncProgressServic block: () -> T): T { reporter?.startTask(nameRes, totalProgress, parentWeight) return block().also { - reporter?.endTask(nameRes) + reporter?.endTask() } } internal inline fun Map.mapWithProgress(reporter: DefaultInitialSyncProgressService?, - taskId: Int, + @StringRes nameRes: Int, parentWeight: Float, transform: (Map.Entry) -> R): List { var current = 0F - reporter?.startTask(taskId, count() + 1, parentWeight) + reporter?.startTask(nameRes, count() + 1, parentWeight) return map { reporter?.reportProgress(current) current++ transform.invoke(it) }.also { - reporter?.endTask(taskId) + reporter?.endTask() } } From b870a8b791c040afdca121cb16e6487dad74a08d Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Tue, 9 Feb 2021 10:30:11 +0100 Subject: [PATCH 12/17] Create ProgressReporter interface --- .../DefaultInitialSyncProgressService.kt | 28 +++++++++++++------ .../session/sync/CryptoSyncHandler.kt | 8 +++--- .../internal/session/sync/GroupSyncHandler.kt | 14 ++++------ .../internal/session/sync/RoomSyncHandler.kt | 16 +++++------ .../session/sync/SyncResponseHandler.kt | 6 ++-- 5 files changed, 41 insertions(+), 31 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt index fb7ba7c58f..e74d905436 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt @@ -22,8 +22,20 @@ import org.matrix.android.sdk.api.session.InitialSyncProgressService import timber.log.Timber import javax.inject.Inject +internal interface ProgressReporter { + fun startTask(@StringRes nameRes: Int, + totalProgress: Int, + parentWeight: Float) + + fun reportProgress(progress: Float) + + fun endTask() +} + @SessionScope -internal class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgressService { +internal class DefaultInitialSyncProgressService @Inject constructor() + : InitialSyncProgressService, + ProgressReporter { private val status = MutableLiveData() @@ -33,9 +45,9 @@ internal class DefaultInitialSyncProgressService @Inject constructor() : Initial return status } - fun startTask(@StringRes nameRes: Int, - totalProgress: Int, - parentWeight: Float) { + override fun startTask(@StringRes nameRes: Int, + totalProgress: Int, + parentWeight: Float) { // Create a rootTask, or add a child to the leaf if (rootTask == null) { rootTask = TaskInfo(nameRes, totalProgress, null, 1F) @@ -51,7 +63,7 @@ internal class DefaultInitialSyncProgressService @Inject constructor() : Initial reportProgress(0F) } - fun reportProgress(progress: Float) { + override fun reportProgress(progress: Float) { rootTask?.let { root -> root.leaf().let { leaf -> // Update the progress of the leaf and all its parents @@ -62,7 +74,7 @@ internal class DefaultInitialSyncProgressService @Inject constructor() : Initial } } - fun endTask() { + override fun endTask() { rootTask?.leaf()?.let { endedTask -> // Ensure the task progress is complete reportProgress(endedTask.totalProgress.toFloat()) @@ -117,7 +129,7 @@ private class TaskInfo(@StringRes val nameRes: Int, } } -internal inline fun reportSubtask(reporter: DefaultInitialSyncProgressService?, +internal inline fun reportSubtask(reporter: ProgressReporter?, @StringRes nameRes: Int, totalProgress: Int, parentWeight: Float, @@ -128,7 +140,7 @@ internal inline fun reportSubtask(reporter: DefaultInitialSyncProgressServic } } -internal inline fun Map.mapWithProgress(reporter: DefaultInitialSyncProgressService?, +internal inline fun Map.mapWithProgress(reporter: ProgressReporter?, @StringRes nameRes: Int, parentWeight: Float, transform: (Map.Entry) -> R): List { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt index 42477e1dd8..a244e7a273 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt @@ -26,7 +26,7 @@ import org.matrix.android.sdk.internal.crypto.MXEventDecryptionResult import org.matrix.android.sdk.internal.crypto.algorithms.olm.OlmDecryptionResult import org.matrix.android.sdk.internal.crypto.model.event.OlmEventContent import org.matrix.android.sdk.internal.crypto.verification.DefaultVerificationService -import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService +import org.matrix.android.sdk.internal.session.ProgressReporter import org.matrix.android.sdk.internal.session.sync.model.SyncResponse import org.matrix.android.sdk.internal.session.sync.model.ToDeviceSyncResponse import timber.log.Timber @@ -35,10 +35,10 @@ import javax.inject.Inject internal class CryptoSyncHandler @Inject constructor(private val cryptoService: DefaultCryptoService, private val verificationService: DefaultVerificationService) { - fun handleToDevice(toDevice: ToDeviceSyncResponse, initialSyncProgressService: DefaultInitialSyncProgressService? = null) { + fun handleToDevice(toDevice: ToDeviceSyncResponse, progressReporter: ProgressReporter? = null) { val total = toDevice.events?.size ?: 0 toDevice.events?.forEachIndexed { index, event -> - initialSyncProgressService?.reportProgress(index * 100F / total) + progressReporter?.reportProgress(index * 100F / total) // Decrypt event if necessary Timber.i("## CRYPTO | To device event from ${event.senderId} of type:${event.type}") decryptToDeviceEvent(event, null) @@ -75,7 +75,7 @@ internal class CryptoSyncHandler @Inject constructor(private val cryptoService: // try to find device id to ease log reading val deviceId = cryptoService.getCryptoDeviceInfo(event.senderId!!).firstOrNull { it.identityKey() == senderKey - }?.deviceId ?: senderKey + }?.deviceId ?: senderKey Timber.e("## CRYPTO | Failed to decrypt to device event from ${event.senderId}|$deviceId reason:<${event.mCryptoError ?: exception}>") } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt index 135f711a6c..03ad76e19a 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt @@ -16,17 +16,17 @@ package org.matrix.android.sdk.internal.session.sync +import io.realm.Realm import org.matrix.android.sdk.R import org.matrix.android.sdk.api.session.room.model.Membership import org.matrix.android.sdk.internal.database.model.GroupEntity import org.matrix.android.sdk.internal.database.model.GroupSummaryEntity import org.matrix.android.sdk.internal.database.query.getOrCreate import org.matrix.android.sdk.internal.database.query.where -import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService +import org.matrix.android.sdk.internal.session.ProgressReporter import org.matrix.android.sdk.internal.session.mapWithProgress import org.matrix.android.sdk.internal.session.sync.model.GroupsSyncResponse import org.matrix.android.sdk.internal.session.sync.model.InvitedGroupSync -import io.realm.Realm import javax.inject.Inject internal class GroupSyncHandler @Inject constructor() { @@ -37,11 +37,9 @@ internal class GroupSyncHandler @Inject constructor() { data class LEFT(val data: Map) : HandlingStrategy() } - fun handle( - realm: Realm, - roomsSyncResponse: GroupsSyncResponse, - reporter: DefaultInitialSyncProgressService? = null - ) { + fun handle(realm: Realm, + roomsSyncResponse: GroupsSyncResponse, + reporter: ProgressReporter? = null) { handleGroupSync(realm, HandlingStrategy.JOINED(roomsSyncResponse.join), reporter) handleGroupSync(realm, HandlingStrategy.INVITED(roomsSyncResponse.invite), reporter) handleGroupSync(realm, HandlingStrategy.LEFT(roomsSyncResponse.leave), reporter) @@ -49,7 +47,7 @@ internal class GroupSyncHandler @Inject constructor() { // PRIVATE METHODS ***************************************************************************** - private fun handleGroupSync(realm: Realm, handlingStrategy: HandlingStrategy, reporter: DefaultInitialSyncProgressService?) { + private fun handleGroupSync(realm: Realm, handlingStrategy: HandlingStrategy, reporter: ProgressReporter?) { val groups = when (handlingStrategy) { is HandlingStrategy.JOINED -> handlingStrategy.data.mapWithProgress(reporter, R.string.initial_sync_start_importing_account_groups, 0.6f) { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt index ef2212cdab..7d8966579e 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt @@ -49,7 +49,7 @@ import org.matrix.android.sdk.internal.database.query.where import org.matrix.android.sdk.internal.di.MoshiProvider import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.extensions.clearWith -import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService +import org.matrix.android.sdk.internal.session.ProgressReporter import org.matrix.android.sdk.internal.session.mapWithProgress import org.matrix.android.sdk.internal.session.reportSubtask import org.matrix.android.sdk.internal.session.room.membership.RoomChangeMembershipStateDataSource @@ -86,12 +86,10 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle data class LEFT(val data: Map) : HandlingStrategy() } - fun handle( - realm: Realm, - roomsSyncResponse: RoomsSyncResponse, - isInitialSync: Boolean, - reporter: DefaultInitialSyncProgressService? = null - ) { + fun handle(realm: Realm, + roomsSyncResponse: RoomsSyncResponse, + isInitialSync: Boolean, + reporter: ProgressReporter? = null) { Timber.v("Execute transaction from $this") handleRoomSync(realm, HandlingStrategy.JOINED(roomsSyncResponse.join), isInitialSync, reporter) handleRoomSync(realm, HandlingStrategy.INVITED(roomsSyncResponse.invite), isInitialSync, reporter) @@ -100,7 +98,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle // PRIVATE METHODS ***************************************************************************** - private fun handleRoomSync(realm: Realm, handlingStrategy: HandlingStrategy, isInitialSync: Boolean, reporter: DefaultInitialSyncProgressService?) { + private fun handleRoomSync(realm: Realm, handlingStrategy: HandlingStrategy, isInitialSync: Boolean, reporter: ProgressReporter?) { val insertType = if (isInitialSync) { EventInsertType.INITIAL_SYNC } else { @@ -137,7 +135,7 @@ internal class RoomSyncHandler @Inject constructor(private val readReceiptHandle handlingStrategy: HandlingStrategy.JOINED, insertType: EventInsertType, syncLocalTimeStampMillis: Long, - reporter: DefaultInitialSyncProgressService?) { + reporter: ProgressReporter?) { val maxSize = (initialSyncStrategy as? InitialSyncStrategy.Optimized)?.maxRoomsToInsert ?: Int.MAX_VALUE val listSize = handlingStrategy.data.keys.size val numberOfChunks = ceil(listSize / maxSize.toDouble()).toInt() diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt index c9376eb9f2..a80b733dcc 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt @@ -25,7 +25,7 @@ import org.matrix.android.sdk.internal.crypto.DefaultCryptoService import org.matrix.android.sdk.internal.di.SessionDatabase import org.matrix.android.sdk.internal.di.SessionId import org.matrix.android.sdk.internal.di.WorkManagerProvider -import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService +import org.matrix.android.sdk.internal.session.ProgressReporter import org.matrix.android.sdk.internal.session.group.GetGroupDataWorker import org.matrix.android.sdk.internal.session.notification.ProcessEventForPushTask import org.matrix.android.sdk.internal.session.reportSubtask @@ -53,7 +53,9 @@ internal class SyncResponseHandler @Inject constructor(@SessionDatabase private private val processEventForPushTask: ProcessEventForPushTask, private val pushRuleService: PushRuleService) { - suspend fun handleResponse(syncResponse: SyncResponse, fromToken: String?, reporter: DefaultInitialSyncProgressService?) { + suspend fun handleResponse(syncResponse: SyncResponse, + fromToken: String?, + reporter: ProgressReporter?) { val isInitialSync = fromToken == null Timber.v("Start handling sync, is InitialSync: $isInitialSync") From 649edffa30cbdadf5d2009f5072841e84160220f Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Tue, 9 Feb 2021 11:04:47 +0100 Subject: [PATCH 13/17] Hide menu if not in developer mode --- .../main/java/im/vector/app/features/home/HomeActivity.kt | 7 +++++++ vector/src/main/res/menu/home.xml | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt b/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt index 2d884f1ba7..2a0ef0bdd6 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt @@ -21,6 +21,7 @@ import android.content.Intent import android.net.Uri import android.os.Bundle import android.os.Parcelable +import android.view.Menu import android.view.MenuItem import androidx.appcompat.app.AlertDialog import androidx.appcompat.widget.Toolbar @@ -362,6 +363,12 @@ class HomeActivity : override fun getMenuRes() = R.menu.home + override fun onPrepareOptionsMenu(menu: Menu): Boolean { + menu.findItem(R.id.menu_home_init_sync_legacy)?.isVisible = vectorPreferences.developerMode() + menu.findItem(R.id.menu_home_init_sync_optimized)?.isVisible = vectorPreferences.developerMode() + return super.onPrepareOptionsMenu(menu) + } + override fun onOptionsItemSelected(item: MenuItem): Boolean { when (item.itemId) { R.id.menu_home_suggestion -> { diff --git a/vector/src/main/res/menu/home.xml b/vector/src/main/res/menu/home.xml index 66cbf53bfd..463b8a9e0d 100644 --- a/vector/src/main/res/menu/home.xml +++ b/vector/src/main/res/menu/home.xml @@ -21,12 +21,12 @@ Date: Tue, 9 Feb 2021 11:36:04 +0100 Subject: [PATCH 14/17] Move to dedicated package and split into several files --- .../matrix/android/sdk/api/session/Session.kt | 1 + .../InitialSyncProgressService.kt | 4 +- .../sdk/internal/session/DefaultSession.kt | 2 +- .../sdk/internal/session/SessionModule.kt | 3 +- .../DefaultInitialSyncProgressService.kt | 78 +------------------ .../internal/session/initsync/Extensions.kt | 45 +++++++++++ .../session/initsync/ProgressReporter.kt | 29 +++++++ .../sdk/internal/session/initsync/TaskInfo.kt | 54 +++++++++++++ .../session/sync/CryptoSyncHandler.kt | 2 +- .../internal/session/sync/GroupSyncHandler.kt | 4 +- .../internal/session/sync/RoomSyncHandler.kt | 6 +- .../session/sync/SyncResponseHandler.kt | 4 +- .../sdk/internal/session/sync/SyncTask.kt | 4 +- .../vector/app/features/home/HomeActivity.kt | 2 +- .../features/home/HomeActivityViewModel.kt | 4 +- .../features/home/HomeActivityViewState.kt | 2 +- 16 files changed, 152 insertions(+), 92 deletions(-) rename matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/{ => initsync}/InitialSyncProgressService.kt (90%) rename matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/{ => initsync}/DefaultInitialSyncProgressService.kt (54%) create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/Extensions.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/ProgressReporter.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/TaskInfo.kt diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt index 86ac0056e2..7a24ccac11 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/Session.kt @@ -35,6 +35,7 @@ import org.matrix.android.sdk.api.session.file.FileService import org.matrix.android.sdk.api.session.group.GroupService import org.matrix.android.sdk.api.session.homeserver.HomeServerCapabilitiesService import org.matrix.android.sdk.api.session.identity.IdentityService +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService import org.matrix.android.sdk.api.session.integrationmanager.IntegrationManagerService import org.matrix.android.sdk.api.session.media.MediaService import org.matrix.android.sdk.api.session.permalinks.PermalinkService diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/InitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/initsync/InitialSyncProgressService.kt similarity index 90% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/InitialSyncProgressService.kt rename to matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/initsync/InitialSyncProgressService.kt index 09c6b8e94c..0953696bc1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/InitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/initsync/InitialSyncProgressService.kt @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.matrix.android.sdk.api.session +package org.matrix.android.sdk.api.session.initsync import androidx.annotation.StringRes import androidx.lifecycle.LiveData diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt index 06bb4bd929..1204a9ccac 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultSession.kt @@ -24,7 +24,7 @@ import org.matrix.android.sdk.api.auth.data.SessionParams import org.matrix.android.sdk.api.failure.GlobalError import org.matrix.android.sdk.api.federation.FederationService import org.matrix.android.sdk.api.pushrules.PushRuleService -import org.matrix.android.sdk.api.session.InitialSyncProgressService +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.account.AccountService import org.matrix.android.sdk.api.session.accountdata.AccountDataService diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt index 468c193ad3..57c2336331 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt @@ -32,7 +32,7 @@ import org.matrix.android.sdk.api.auth.data.HomeServerConnectionConfig import org.matrix.android.sdk.api.auth.data.SessionParams import org.matrix.android.sdk.api.auth.data.sessionId import org.matrix.android.sdk.api.crypto.MXCryptoConfig -import org.matrix.android.sdk.api.session.InitialSyncProgressService +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.accountdata.AccountDataService import org.matrix.android.sdk.api.session.homeserver.HomeServerCapabilitiesService @@ -77,6 +77,7 @@ import org.matrix.android.sdk.internal.session.call.CallEventProcessor import org.matrix.android.sdk.internal.session.download.DownloadProgressInterceptor import org.matrix.android.sdk.internal.session.homeserver.DefaultHomeServerCapabilitiesService import org.matrix.android.sdk.internal.session.identity.DefaultIdentityService +import org.matrix.android.sdk.internal.session.initsync.DefaultInitialSyncProgressService import org.matrix.android.sdk.internal.session.integrationmanager.IntegrationManager import org.matrix.android.sdk.internal.session.permalinks.DefaultPermalinkService import org.matrix.android.sdk.internal.session.room.EventRelationsAggregationProcessor diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt similarity index 54% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt rename to matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt index e74d905436..fd3c629866 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/DefaultInitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,25 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.matrix.android.sdk.internal.session +package org.matrix.android.sdk.internal.session.initsync import androidx.annotation.StringRes import androidx.lifecycle.LiveData import androidx.lifecycle.MutableLiveData -import org.matrix.android.sdk.api.session.InitialSyncProgressService -import timber.log.Timber +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService +import org.matrix.android.sdk.internal.session.SessionScope import javax.inject.Inject -internal interface ProgressReporter { - fun startTask(@StringRes nameRes: Int, - totalProgress: Int, - parentWeight: Float) - - fun reportProgress(progress: Float) - - fun endTask() -} - @SessionScope internal class DefaultInitialSyncProgressService @Inject constructor() : InitialSyncProgressService, @@ -94,63 +84,3 @@ internal class DefaultInitialSyncProgressService @Inject constructor() status.postValue(InitialSyncProgressService.Status.Idle) } } - -private class TaskInfo(@StringRes val nameRes: Int, - val totalProgress: Int, - val parent: TaskInfo?, - val parentWeight: Float) { - var child: TaskInfo? = null - var currentProgress = 0F - private set - private val offset = parent?.currentProgress ?: 0F - - /** - * Get the further child - */ - fun leaf(): TaskInfo { - var last = this - while (last.child != null) { - last = last.child!! - } - return last - } - - /** - * Set progress of this task and update the parent progress iteratively - */ - fun setProgress(progress: Float) { - Timber.v("setProgress: $progress / $totalProgress") - currentProgress = progress - - parent?.let { - val parentProgress = (currentProgress / totalProgress) * (parentWeight * it.totalProgress) - it.setProgress(offset + parentProgress) - } - } -} - -internal inline fun reportSubtask(reporter: ProgressReporter?, - @StringRes nameRes: Int, - totalProgress: Int, - parentWeight: Float, - block: () -> T): T { - reporter?.startTask(nameRes, totalProgress, parentWeight) - return block().also { - reporter?.endTask() - } -} - -internal inline fun Map.mapWithProgress(reporter: ProgressReporter?, - @StringRes nameRes: Int, - parentWeight: Float, - transform: (Map.Entry) -> R): List { - var current = 0F - reporter?.startTask(nameRes, count() + 1, parentWeight) - return map { - reporter?.reportProgress(current) - current++ - transform.invoke(it) - }.also { - reporter?.endTask() - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/Extensions.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/Extensions.kt new file mode 100644 index 0000000000..f58559117c --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/Extensions.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2021 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.initsync + +import androidx.annotation.StringRes + +internal inline fun reportSubtask(reporter: ProgressReporter?, + @StringRes nameRes: Int, + totalProgress: Int, + parentWeight: Float, + block: () -> T): T { + reporter?.startTask(nameRes, totalProgress, parentWeight) + return block().also { + reporter?.endTask() + } +} + +internal inline fun Map.mapWithProgress(reporter: ProgressReporter?, + @StringRes nameRes: Int, + parentWeight: Float, + transform: (Map.Entry) -> R): List { + var current = 0F + reporter?.startTask(nameRes, count() + 1, parentWeight) + return map { + reporter?.reportProgress(current) + current++ + transform.invoke(it) + }.also { + reporter?.endTask() + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/ProgressReporter.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/ProgressReporter.kt new file mode 100644 index 0000000000..5361d107d2 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/ProgressReporter.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2021 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.initsync + +import androidx.annotation.StringRes + +internal interface ProgressReporter { + fun startTask(@StringRes nameRes: Int, + totalProgress: Int, + parentWeight: Float) + + fun reportProgress(progress: Float) + + fun endTask() +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/TaskInfo.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/TaskInfo.kt new file mode 100644 index 0000000000..37c2b152a9 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/TaskInfo.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2021 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.initsync + +import androidx.annotation.StringRes +import timber.log.Timber + +internal class TaskInfo(@StringRes val nameRes: Int, + val totalProgress: Int, + val parent: TaskInfo?, + val parentWeight: Float) { + var child: TaskInfo? = null + var currentProgress = 0F + private set + private val offset = parent?.currentProgress ?: 0F + + /** + * Get the further child + */ + fun leaf(): TaskInfo { + var last = this + while (last.child != null) { + last = last.child!! + } + return last + } + + /** + * Set progress of this task and update the parent progress iteratively + */ + fun setProgress(progress: Float) { + Timber.v("setProgress: $progress / $totalProgress") + currentProgress = progress + + parent?.let { + val parentProgress = (currentProgress / totalProgress) * (parentWeight * it.totalProgress) + it.setProgress(offset + parentProgress) + } + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt index a244e7a273..ae60faf905 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/CryptoSyncHandler.kt @@ -26,7 +26,7 @@ import org.matrix.android.sdk.internal.crypto.MXEventDecryptionResult import org.matrix.android.sdk.internal.crypto.algorithms.olm.OlmDecryptionResult import org.matrix.android.sdk.internal.crypto.model.event.OlmEventContent import org.matrix.android.sdk.internal.crypto.verification.DefaultVerificationService -import org.matrix.android.sdk.internal.session.ProgressReporter +import org.matrix.android.sdk.internal.session.initsync.ProgressReporter import org.matrix.android.sdk.internal.session.sync.model.SyncResponse import org.matrix.android.sdk.internal.session.sync.model.ToDeviceSyncResponse import timber.log.Timber diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt index 03ad76e19a..112236b14f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/GroupSyncHandler.kt @@ -23,8 +23,8 @@ import org.matrix.android.sdk.internal.database.model.GroupEntity import org.matrix.android.sdk.internal.database.model.GroupSummaryEntity import org.matrix.android.sdk.internal.database.query.getOrCreate import org.matrix.android.sdk.internal.database.query.where -import org.matrix.android.sdk.internal.session.ProgressReporter -import org.matrix.android.sdk.internal.session.mapWithProgress +import org.matrix.android.sdk.internal.session.initsync.ProgressReporter +import org.matrix.android.sdk.internal.session.initsync.mapWithProgress import org.matrix.android.sdk.internal.session.sync.model.GroupsSyncResponse import org.matrix.android.sdk.internal.session.sync.model.InvitedGroupSync import javax.inject.Inject diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt index 7d8966579e..648dd2d88f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/RoomSyncHandler.kt @@ -49,9 +49,9 @@ import org.matrix.android.sdk.internal.database.query.where import org.matrix.android.sdk.internal.di.MoshiProvider import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.extensions.clearWith -import org.matrix.android.sdk.internal.session.ProgressReporter -import org.matrix.android.sdk.internal.session.mapWithProgress -import org.matrix.android.sdk.internal.session.reportSubtask +import org.matrix.android.sdk.internal.session.initsync.ProgressReporter +import org.matrix.android.sdk.internal.session.initsync.mapWithProgress +import org.matrix.android.sdk.internal.session.initsync.reportSubtask import org.matrix.android.sdk.internal.session.room.membership.RoomChangeMembershipStateDataSource import org.matrix.android.sdk.internal.session.room.membership.RoomMemberEventHandler import org.matrix.android.sdk.internal.session.room.read.FullyReadContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt index a80b733dcc..3f501858b1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncResponseHandler.kt @@ -25,10 +25,10 @@ import org.matrix.android.sdk.internal.crypto.DefaultCryptoService import org.matrix.android.sdk.internal.di.SessionDatabase import org.matrix.android.sdk.internal.di.SessionId import org.matrix.android.sdk.internal.di.WorkManagerProvider -import org.matrix.android.sdk.internal.session.ProgressReporter +import org.matrix.android.sdk.internal.session.initsync.ProgressReporter import org.matrix.android.sdk.internal.session.group.GetGroupDataWorker import org.matrix.android.sdk.internal.session.notification.ProcessEventForPushTask -import org.matrix.android.sdk.internal.session.reportSubtask +import org.matrix.android.sdk.internal.session.initsync.reportSubtask import org.matrix.android.sdk.internal.session.sync.model.GroupsSyncResponse import org.matrix.android.sdk.internal.session.sync.model.RoomsSyncResponse import org.matrix.android.sdk.internal.session.sync.model.SyncResponse diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt index ea06db7071..bc63b991b2 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt @@ -24,10 +24,10 @@ import org.matrix.android.sdk.internal.network.GlobalErrorReceiver import org.matrix.android.sdk.internal.network.TimeOutInterceptor import org.matrix.android.sdk.internal.network.executeRequest import org.matrix.android.sdk.internal.network.toFailure -import org.matrix.android.sdk.internal.session.DefaultInitialSyncProgressService +import org.matrix.android.sdk.internal.session.initsync.DefaultInitialSyncProgressService import org.matrix.android.sdk.internal.session.filter.FilterRepository import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask -import org.matrix.android.sdk.internal.session.reportSubtask +import org.matrix.android.sdk.internal.session.initsync.reportSubtask import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync import org.matrix.android.sdk.internal.session.sync.model.SyncResponse import org.matrix.android.sdk.internal.session.sync.parsing.InitialSyncResponseParser diff --git a/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt b/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt index 2a0ef0bdd6..74399d9215 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeActivity.kt @@ -60,7 +60,7 @@ import im.vector.app.features.workers.signout.ServerBackupStatusViewState import im.vector.app.push.fcm.FcmHelper import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.parcelize.Parcelize -import org.matrix.android.sdk.api.session.InitialSyncProgressService +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService import org.matrix.android.sdk.api.session.permalinks.PermalinkService import org.matrix.android.sdk.api.util.MatrixItem import org.matrix.android.sdk.internal.session.sync.InitialSyncStrategy diff --git a/vector/src/main/java/im/vector/app/features/home/HomeActivityViewModel.kt b/vector/src/main/java/im/vector/app/features/home/HomeActivityViewModel.kt index 62bdc61b63..ad61928509 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeActivityViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeActivityViewModel.kt @@ -40,7 +40,7 @@ import org.matrix.android.sdk.api.auth.registration.RegistrationFlowResponse import org.matrix.android.sdk.api.auth.registration.nextUncompletedStage import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.pushrules.RuleIds -import org.matrix.android.sdk.api.session.InitialSyncProgressService +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService import org.matrix.android.sdk.api.session.room.model.Membership import org.matrix.android.sdk.api.session.room.roomSummaryQueryParams import org.matrix.android.sdk.api.util.toMatrixItem @@ -130,7 +130,7 @@ class HomeActivityViewModel @AssistedInject constructor( // Schedule a check of the bootstrap when the init sync will be finished checkBootstrap = true } - is InitialSyncProgressService.Status.Idle -> { + is InitialSyncProgressService.Status.Idle -> { if (checkBootstrap) { checkBootstrap = false maybeBootstrapCrossSigningAfterInitialSync() diff --git a/vector/src/main/java/im/vector/app/features/home/HomeActivityViewState.kt b/vector/src/main/java/im/vector/app/features/home/HomeActivityViewState.kt index 0f80fa29ef..d4df7cd073 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeActivityViewState.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeActivityViewState.kt @@ -17,7 +17,7 @@ package im.vector.app.features.home import com.airbnb.mvrx.MvRxState -import org.matrix.android.sdk.api.session.InitialSyncProgressService +import org.matrix.android.sdk.api.session.initsync.InitialSyncProgressService data class HomeActivityViewState( val initialSyncProgressServiceStatus: InitialSyncProgressService.Status = InitialSyncProgressService.Status.Idle From c3192d9f7cc5b8c44b7f252c63b86b962aa56634 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Tue, 9 Feb 2021 11:41:26 +0100 Subject: [PATCH 15/17] Latest improvement --- .../DefaultInitialSyncProgressService.kt | 32 ++++++++++++------- .../sdk/internal/session/sync/SyncTask.kt | 5 ++- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt index fd3c629866..dc069d3e5e 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/initsync/DefaultInitialSyncProgressService.kt @@ -35,21 +35,29 @@ internal class DefaultInitialSyncProgressService @Inject constructor() return status } + /** + * Create a rootTask + */ + fun startRoot(@StringRes nameRes: Int, + totalProgress: Int) { + endAll() + rootTask = TaskInfo(nameRes, totalProgress, null, 1F) + reportProgress(0F) + } + + /** + * Add a child to the leaf + */ override fun startTask(@StringRes nameRes: Int, totalProgress: Int, parentWeight: Float) { - // Create a rootTask, or add a child to the leaf - if (rootTask == null) { - rootTask = TaskInfo(nameRes, totalProgress, null, 1F) - } else { - val currentLeaf = rootTask!!.leaf() - currentLeaf.child = TaskInfo( - nameRes = nameRes, - totalProgress = totalProgress, - parent = currentLeaf, - parentWeight = parentWeight - ) - } + val currentLeaf = rootTask?.leaf() ?: return + currentLeaf.child = TaskInfo( + nameRes = nameRes, + totalProgress = totalProgress, + parent = currentLeaf, + parentWeight = parentWeight + ) reportProgress(0F) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt index bc63b991b2..dea047f7d0 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/sync/SyncTask.kt @@ -24,9 +24,9 @@ import org.matrix.android.sdk.internal.network.GlobalErrorReceiver import org.matrix.android.sdk.internal.network.TimeOutInterceptor import org.matrix.android.sdk.internal.network.executeRequest import org.matrix.android.sdk.internal.network.toFailure -import org.matrix.android.sdk.internal.session.initsync.DefaultInitialSyncProgressService import org.matrix.android.sdk.internal.session.filter.FilterRepository import org.matrix.android.sdk.internal.session.homeserver.GetHomeServerCapabilitiesTask +import org.matrix.android.sdk.internal.session.initsync.DefaultInitialSyncProgressService import org.matrix.android.sdk.internal.session.initsync.reportSubtask import org.matrix.android.sdk.internal.session.sync.model.LazyRoomSync import org.matrix.android.sdk.internal.session.sync.model.SyncResponse @@ -90,8 +90,7 @@ internal class DefaultSyncTask @Inject constructor( if (isInitialSync) { // We might want to get the user information in parallel too userStore.createOrUpdate(userId) - initialSyncProgressService.endAll() - initialSyncProgressService.startTask(R.string.initial_sync_start_importing_account, 100, 1F) + initialSyncProgressService.startRoot(R.string.initial_sync_start_importing_account, 100) } // Maybe refresh the home server capabilities data we know getHomeServerCapabilitiesTask.execute(GetHomeServerCapabilitiesTask.Params(forceRefresh = false)) From 29359ad119f96cf58563eef0851a31b5152fe650 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Tue, 16 Feb 2021 12:04:00 +0100 Subject: [PATCH 16/17] Update change after rebase --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 930068dd6c..9d1ba6ce07 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,7 @@ Features ✨: Improvements 🙌: - Fetch homeserver type and version and display in a new setting screen and add info in rageshakes (#2831) + - Improve initial sync performance (#983) Bugfix 🐛: - @@ -90,7 +91,6 @@ Improvements 🙌: - SSO support for cross signing (#1062) - Deactivate account when logged in with SSO (#1264) - SSO UIA doesn't work (#2754) - - Improve initial sync performance (#983) Bugfix 🐛: - Fix clear cache issue: sometimes, after a clear cache, there is still a token, so the init sync service is not started. From c181f7e4218e5cfc684723320dd5ad4874727fad Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Wed, 17 Feb 2021 11:06:32 +0100 Subject: [PATCH 17/17] Please code quality check --- tools/check/forbidden_strings_in_code.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check/forbidden_strings_in_code.txt b/tools/check/forbidden_strings_in_code.txt index e9b29d99ba..a5127dc8aa 100644 --- a/tools/check/forbidden_strings_in_code.txt +++ b/tools/check/forbidden_strings_in_code.txt @@ -161,7 +161,7 @@ Formatter\.formatShortFileSize===1 # android\.text\.TextUtils ### This is not a rule, but a warning: the number of "enum class" has changed. For Json classes, it is mandatory that they have `@JsonClass(generateAdapter = false)`. If the enum is not used as a Json class, change the value in file forbidden_strings_in_code.txt -enum class===88 +enum class===89 ### Do not import temporary legacy classes import org.matrix.android.sdk.internal.legacy.riot===3