From 1ac7fc77022df134138ef3531a7b6b7d4ff32815 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Wed, 3 May 2023 11:55:44 +0200 Subject: [PATCH 1/9] Rust Crypto SDK is now the default, and the build will replace the existing application. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b135aa5c9e4..319acb2160b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,7 +79,7 @@ jobs: - name: Execute exodus-standalone uses: docker://exodusprivacy/exodus-standalone:latest with: - args: /github/workspace/gplayRustCrypto/release/vector-gplay-rustCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json + args: /github/workspace/gplayRustCrypto/release/vector-gplay-kotlinRust-universal-release-unsigned.apk -j -o /github/workspace/exodus.json - name: Upload exodus json report uses: actions/upload-artifact@v3 with: From 56d0a1e194b6bb956c354680a0412d7eb7e78a45 Mon Sep 17 00:00:00 2001 From: Benoit Marty Date: Wed, 3 May 2023 14:01:16 +0200 Subject: [PATCH 2/9] Fix typo --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 319acb2160b..b135aa5c9e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,7 +79,7 @@ jobs: - name: Execute exodus-standalone uses: docker://exodusprivacy/exodus-standalone:latest with: - args: /github/workspace/gplayRustCrypto/release/vector-gplay-kotlinRust-universal-release-unsigned.apk -j -o /github/workspace/exodus.json + args: /github/workspace/gplayRustCrypto/release/vector-gplay-rustCrypto-universal-release-unsigned.apk -j -o /github/workspace/exodus.json - name: Upload exodus json report uses: actions/upload-artifact@v3 with: From 6aaf5747b9871d7bce6fab5db3180ee9933e49d6 Mon Sep 17 00:00:00 2001 From: valere Date: Sat, 6 May 2023 09:48:01 +0200 Subject: [PATCH 3/9] create rust db as a realm migration --- .../database/CryptoSanityMigrationTest.kt | 30 ++- ...cElementAndroidToElementRMigrationTest.kt} | 62 +++-- .../store/db/RealmCryptoStoreMigration.kt | 0 .../session/MigrateEAtoEROperation.kt | 31 --- .../sdk/internal/session/SessionModule.kt | 15 +- .../store/db/RealmCryptoStoreMigration.kt | 95 +++++++ .../store/db/migration/MigrateCryptoTo022.kt | 48 ++++ .../rust/ExtractMigrationDataUseCase.kt | 91 ++----- .../store/db/migration/rust/ExtractUtils.kt | 244 ++++++++++++++++++ .../session/MigrateEAtoEROperation.kt | 26 +- 10 files changed, 491 insertions(+), 151 deletions(-) rename matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/{ElementAndroidToElementRMigrationTest.kt => DynamicElementAndroidToElementRMigrationTest.kt} (63%) rename matrix-sdk-android/src/{main => kotlinCrypto}/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt (100%) delete mode 100644 matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt diff --git a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt index 2643bf643a0..762129a3f70 100644 --- a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt +++ b/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -18,14 +18,20 @@ package org.matrix.android.sdk.internal.database import android.content.Context import androidx.test.platform.app.InstrumentationRegistry +import io.mockk.spyk import io.realm.Realm import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import org.matrix.android.sdk.TestBuildVersionSdkIntProvider +import org.matrix.android.sdk.api.securestorage.SecretStoringUtils +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule import org.matrix.android.sdk.internal.util.time.Clock +import java.io.File +import java.security.KeyStore class CryptoSanityMigrationTest { @get:Rule val configurationFactory = TestRealmConfigurationFactory() @@ -43,14 +49,28 @@ class CryptoSanityMigrationTest { realm?.close() } + private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) } + @Test fun cryptoDatabaseShouldMigrateGracefully() { val realmName = "crypto_store_20.realm" - val migration = RealmCryptoStoreMigration(object : Clock { - override fun epochMillis(): Long { - return 0L - } - }) + + val migration = RealmCryptoStoreMigration( + object : Clock { + override fun epochMillis(): Long { + return 0L + } + }, + RustEncryptionConfiguration( + "foo", + RealmKeysUtils( + context, + SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) + ) + ), + File("test_rust") + ) + val realmConfiguration = configurationFactory.createConfiguration( realmName, "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca", diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt similarity index 63% rename from matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt rename to matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt index a0847a7ad3e..e6f027d4083 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt @@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration import android.content.Context import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry +import io.mockk.spyk import io.realm.Realm import io.realm.kotlin.where import org.amshove.kluent.internal.assertEquals @@ -31,32 +32,33 @@ import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith import org.matrix.android.sdk.InstrumentedTest +import org.matrix.android.sdk.TestBuildVersionSdkIntProvider +import org.matrix.android.sdk.api.securestorage.SecretStoringUtils +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields +import org.matrix.android.sdk.internal.database.RealmKeysUtils import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory -import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation import org.matrix.android.sdk.internal.util.time.Clock import org.matrix.olm.OlmAccount import org.matrix.olm.OlmManager import org.matrix.rustcomponents.sdk.crypto.OlmMachine import java.io.File +import java.security.KeyStore @RunWith(AndroidJUnit4::class) -class ElementAndroidToElementRMigrationTest : InstrumentedTest { +class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest { @get:Rule val configurationFactory = TestRealmConfigurationFactory() - lateinit var context: Context + var context: Context = InstrumentationRegistry.getInstrumentation().context var realm: Realm? = null @Before fun setUp() { // Ensure Olm is initialized OlmManager() - context = InstrumentationRegistry.getInstrumentation().context } @After @@ -64,7 +66,22 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { realm?.close() } + private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) } + + private val rustEncryptionConfiguration = RustEncryptionConfiguration( + "foo", + RealmKeysUtils( + context, + SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) + ) + ) + + private val fakeClock = object : Clock { + override fun epochMillis() = 0L + } + @Test +<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() { testMigrate(false) } @@ -76,10 +93,13 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { } private fun testMigrate(migrateGroupSessions: Boolean) { +======= + fun dynamic_migration_to_rust() { + val targetFile = File(configurationFactory.root, "rust-sdk") + +>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt val realmName = "crypto_store_migration_16.realm" - val migration = RealmCryptoStoreMigration(object : Clock { - override fun epochMillis() = 0L - }) + val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile) val realmConfiguration = configurationFactory.createConfiguration( realmName, @@ -91,12 +111,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { configurationFactory.copyRealmFromAssets(context, realmName, realmName) realm = Realm.getInstance(realmConfiguration) - val metaData = realm!!.where().findFirst()!! val userId = metaData.userId!! val deviceId = metaData.deviceId!! val olmAccount = metaData.getOlmAccount()!! +<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt val extractor = MigrateEAtoEROperation(migrateGroupSessions) val targetFile = File(configurationFactory.root, "rust-sdk") @@ -104,6 +124,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { extractor.execute(realmConfiguration, targetFile, null) val machine = OlmMachine(userId, deviceId, targetFile.path, null) +======= + val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase()) +>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"]) assertNotNull(machine.getBackupKeys()) @@ -112,6 +135,7 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { assertTrue(crossSigningStatus.hasSelfSigning) assertTrue(crossSigningStatus.hasUserSigning) +<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt if (migrateGroupSessions) { val inboundGroupSessionEntities = realm!!.where().findAll() assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt()) @@ -122,19 +146,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest { .findAll() assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt()) } +======= + // How to check that olm sessions have been migrated? + // Can see it from logs +>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt } - -// @Test -// fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() { -// val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false) -// Realm.getInstance(realmConfiguration).use { -// assertTrue(it.isEmpty) -// } -// val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null) -// assertNull(machine.getBackupKeys()) -// val crossSigningStatus = machine.crossSigningStatus() -// assertFalse(crossSigningStatus.hasMaster) -// assertFalse(crossSigningStatus.hasSelfSigning) -// assertFalse(crossSigningStatus.hasUserSigning) -// } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt similarity index 100% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt rename to matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt diff --git a/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt deleted file mode 100644 index 3fd6d1ecf10..00000000000 --- a/matrix-sdk-android/src/kotlinCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2021 The Matrix.org Foundation C.I.C. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.matrix.android.sdk.internal.session - -import io.realm.RealmConfiguration -import timber.log.Timber -import java.io.File - -class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) { - - fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File { - // to remove unused warning - Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions") - // no op in kotlinCrypto - return sessionFilesDir - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt index f0aaf8e59eb..4e778e04cf1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/SessionModule.kt @@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService import org.matrix.android.sdk.api.session.typing.TypingUsersTracker import org.matrix.android.sdk.api.util.md5 -import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask @@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory import org.matrix.android.sdk.internal.di.Authenticated import org.matrix.android.sdk.internal.di.CacheDirectory -import org.matrix.android.sdk.internal.di.CryptoDatabase import org.matrix.android.sdk.internal.di.DeviceId import org.matrix.android.sdk.internal.di.SessionDatabase import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory @@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter import retrofit2.Retrofit -import timber.log.Timber import java.io.File import javax.inject.Provider import javax.inject.Qualifier -import kotlin.system.measureTimeMillis @Qualifier @Retention(AnnotationRetention.RUNTIME) @@ -189,17 +185,8 @@ internal abstract class SessionModule { @SessionScope fun providesRustCryptoFilesDir( @SessionFilesDirectory parent: File, - @CryptoDatabase realmConfiguration: RealmConfiguration, - rustEncryptionConfiguration: RustEncryptionConfiguration, ): File { - val target = File(parent, "rustFlavor") - val file: File - measureTimeMillis { - file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase()) - }.let { duration -> - Timber.v("Migrating to ER in $duration ms") - } - return file + return File(parent, "rustFlavor") } @JvmStatic diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt new file mode 100644 index 00000000000..76054a52578 --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt @@ -0,0 +1,95 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db + +import io.realm.DynamicRealm +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021 +import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022 +import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory +import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration +import org.matrix.android.sdk.internal.util.time.Clock +import java.io.File +import javax.inject.Inject + +/** + * Schema version history: + * 0, 1, 2: legacy Riot-Android; + * 3: migrate to RiotX schema; + * 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6). + */ +internal class RealmCryptoStoreMigration @Inject constructor( + private val clock: Clock, + private val rustEncryptionConfiguration: RustEncryptionConfiguration, + @SessionRustFilesDirectory + private val rustDirectory: File, +) : MatrixRealmMigration( + dbName = "Crypto", + schemaVersion = 22L, +) { + /** + * Forces all RealmCryptoStoreMigration instances to be equal. + * Avoids Realm throwing when multiple instances of the migration are set. + */ + override fun equals(other: Any?) = other is RealmCryptoStoreMigration + override fun hashCode() = 5000 + + override fun doMigrate(realm: DynamicRealm, oldVersion: Long) { + if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform() + if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform() + if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform() + if (oldVersion < 4) MigrateCryptoTo004(realm).perform() + if (oldVersion < 5) MigrateCryptoTo005(realm).perform() + if (oldVersion < 6) MigrateCryptoTo006(realm).perform() + if (oldVersion < 7) MigrateCryptoTo007(realm).perform() + if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform() + if (oldVersion < 9) MigrateCryptoTo009(realm).perform() + if (oldVersion < 10) MigrateCryptoTo010(realm).perform() + if (oldVersion < 11) MigrateCryptoTo011(realm).perform() + if (oldVersion < 12) MigrateCryptoTo012(realm).perform() + if (oldVersion < 13) MigrateCryptoTo013(realm).perform() + if (oldVersion < 14) MigrateCryptoTo014(realm).perform() + if (oldVersion < 15) MigrateCryptoTo015(realm).perform() + if (oldVersion < 16) MigrateCryptoTo016(realm).perform() + if (oldVersion < 17) MigrateCryptoTo017(realm).perform() + if (oldVersion < 18) MigrateCryptoTo018(realm).perform() + if (oldVersion < 19) MigrateCryptoTo019(realm).perform() + if (oldVersion < 20) MigrateCryptoTo020(realm).perform() + if (oldVersion < 21) MigrateCryptoTo021(realm).perform() + if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform() + } +} diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt new file mode 100644 index 00000000000..d26fe017411 --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2023 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db.migration + +import io.realm.DynamicRealm +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation +import org.matrix.android.sdk.internal.util.database.RealmMigrator +import java.io.File + +/** + * This migration creates the rust database and migrates from legacy crypto + */ +internal class MigrateCryptoTo022( + realm: DynamicRealm, + private val rustDirectory: File, + private val rustEncryptionConfiguration: RustEncryptionConfiguration +) : RealmMigrator( + realm, + 22 +) { + override fun doMigrate(realm: DynamicRealm) { + // Migrate to rust! + val migrateOperation = MigrateEAtoEROperation() + migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase()) + + // wa can't delete all for now, but we can do some cleaning + realm.schema.get("OlmSessionEntity")?.transform { + it.deleteFromRealm() + } + + // a future migration will clean the rest + } +} diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt index da125b8e16b..5431f17dae0 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt @@ -24,12 +24,9 @@ import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity import org.matrix.olm.OlmSession import org.matrix.olm.OlmUtility -import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport import org.matrix.rustcomponents.sdk.crypto.MigrationData -import org.matrix.rustcomponents.sdk.crypto.PickledAccount import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession import org.matrix.rustcomponents.sdk.crypto.PickledSession import timber.log.Timber @@ -40,7 +37,7 @@ private val charset = Charset.forName("UTF-8") internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) { - fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) { + fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { return try { extract(realm, importPartial) } catch (failure: Throwable) { @@ -57,89 +54,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f } } - private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) { - val metadataEntity = realm.where().findFirst() - ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity") - + private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { val pickleKey = OlmUtility.getRandomKey() - val masterKey = metadataEntity.xSignMasterPrivateKey - val userKey = metadataEntity.xSignUserPrivateKey - val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey - - val userId = metadataEntity.userId - ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") - val deviceId = metadataEntity.deviceId - ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") - - val backupVersion = metadataEntity.backupVersion - val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey - - val isOlmAccountShared = metadataEntity.deviceKeysSentToServer - - val olmAccount = metadataEntity.getOlmAccount() - ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") - val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() - olmAccount.oneTimeKeys() - val pickledAccount = PickledAccount( - userId = userId, - deviceId = deviceId, - pickle = pickledOlmAccount, - shared = isOlmAccountShared, - uploadedSignedKeyCount = 50 - ) - - val baseExtract = MigrationData( - account = pickledAccount, - pickleKey = pickleKey.map { it.toUByte() }, - crossSigning = CrossSigningKeyExport( - masterKey = masterKey, - selfSigningKey = selfSignedKey, - userSigningKey = userKey - ), - sessions = emptyList(), - backupRecoveryKey = backupRecoveryKey, - trackedUsers = emptyList(), - inboundGroupSessions = emptyList(), - backupVersion = backupVersion, - // TODO import room settings from legacy DB - roomSettings = emptyMap() - ) + val baseExtract = realm.getPickledAccount(pickleKey) // import the account asap importPartial(baseExtract) val chunkSize = 500 - realm.where() - .findAll() - .chunked(chunkSize) { chunk -> - val trackedUserIds = chunk.mapNotNull { it.userId } - importPartial( - baseExtract.copy(trackedUsers = trackedUserIds) - ) - } + realm.trackedUsersChunk(500) { + importPartial( + baseExtract.copy(trackedUsers = it) + ) + } var migratedOlmSessionCount = 0 - var readTime = 0L var writeTime = 0L measureTimeMillis { - realm.where().findAll() - .chunked(chunkSize) { chunk -> - migratedOlmSessionCount += chunk.size - val export: List - measureTimeMillis { - export = chunk.map { it.toPickledSession(pickleKey) } - }.also { - readTime += it - } - measureTimeMillis { - importPartial( - baseExtract.copy(sessions = export) - ) - }.also { writeTime += it } - } + realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions -> + migratedOlmSessionCount += pickledSessions.size + measureTimeMillis { + importPartial( + baseExtract.copy(sessions = pickledSessions) + ) + }.also { writeTime += it } + } }.also { Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions") - Timber.i("Migration: extract time $readTime") Timber.i("Migration: rust import time $writeTime") } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt new file mode 100644 index 00000000000..343960b92bd --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db.migration.rust + +import io.realm.kotlin.where +import okhttp3.internal.toImmutableList +import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm +import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields +import org.matrix.olm.OlmAccount +import org.matrix.olm.OlmSession +import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport +import org.matrix.rustcomponents.sdk.crypto.MigrationData +import org.matrix.rustcomponents.sdk.crypto.PickledAccount +import org.matrix.rustcomponents.sdk.crypto.PickledSession +import java.nio.charset.Charset + +sealed class RealmToMigrate { + data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate() + data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate() +} + +fun RealmToMigrate.hasExistingData(): Boolean { + return when (this) { + is RealmToMigrate.ClassicRealm -> { + !this.realm.isEmpty && + // Check if there is a MetaData object + this.realm.where().count() > 0 && + this.realm.where().findFirst()?.olmAccountData != null + } + is RealmToMigrate.DynamicRealm -> { + return true + } + } +} + +@Throws +fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData { + return when (this) { + is RealmToMigrate.ClassicRealm -> { + val metadataEntity = realm.where().findFirst() + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity") + + val masterKey = metadataEntity.xSignMasterPrivateKey + val userKey = metadataEntity.xSignUserPrivateKey + val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey + + val userId = metadataEntity.userId + ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") + val deviceId = metadataEntity.deviceId + ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") + + val backupVersion = metadataEntity.backupVersion + val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey + + val isOlmAccountShared = metadataEntity.deviceKeysSentToServer + + val olmAccount = metadataEntity.getOlmAccount() + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") + val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() + + val pickledAccount = PickledAccount( + userId = userId, + deviceId = deviceId, + pickle = pickledOlmAccount, + shared = isOlmAccountShared, + uploadedSignedKeyCount = 50 + ) + MigrationData( + account = pickledAccount, + pickleKey = pickleKey.map { it.toUByte() }, + crossSigning = CrossSigningKeyExport( + masterKey = masterKey, + selfSigningKey = selfSignedKey, + userSigningKey = userKey + ), + sessions = emptyList(), + backupRecoveryKey = backupRecoveryKey, + trackedUsers = emptyList(), + inboundGroupSessions = emptyList(), + backupVersion = backupVersion, + // TODO import room settings from legacy DB + roomSettings = emptyMap() + ) + } + is RealmToMigrate.DynamicRealm -> { + val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity") + ?: throw java.lang.IllegalStateException("Missing Metadata entity") + + var migrationData: MigrationData? = null + cryptoMetadataEntitySchema.transform { dynMetaData -> + + val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA) + + val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY) + val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY) + val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY) + + val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID) + ?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null") + val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID) + ?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null") + + val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION) + val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY) + + val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER) + + val olmAccount = deserializeFromRealm(serializedOlmAccount) + ?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account") + + val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString() + + val pickledAccount = PickledAccount( + userId = userId, + deviceId = deviceId, + pickle = pickledOlmAccount, + shared = isOlmAccountShared, + uploadedSignedKeyCount = 50 + ) + + migrationData = MigrationData( + account = pickledAccount, + pickleKey = pickleKey.map { it.toUByte() }, + crossSigning = CrossSigningKeyExport( + masterKey = masterKey, + selfSigningKey = selfSignedKey, + userSigningKey = userKey + ), + sessions = emptyList(), + backupRecoveryKey = backupRecoveryKey, + trackedUsers = emptyList(), + inboundGroupSessions = emptyList(), + backupVersion = backupVersion, + // TODO import room settings from legacy DB + roomSettings = emptyMap() + ) + } + migrationData!! + } + } +} + +fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where() + .findAll() + .chunked(chunkSize) + .onEach { + onChunk(it.mapNotNull { it.userId }) + } + } + is RealmToMigrate.DynamicRealm -> { + val userList = mutableListOf() + realm.schema.get("UserEntity")?.transform { + val userId = it.getString(UserEntityFields.USER_ID) + // should we check the tracking status? + userList.add(userId) + if (userList.size > chunkSize) { + onChunk(userList.toImmutableList()) + userList.clear() + } + } + if (userList.isNotEmpty()) { + onChunk(userList) + } + } + } +} + +fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where().findAll() + .chunked(chunkSize) { chunk -> + val export = chunk.map { it.toPickledSession(pickleKey) } + onChunk(export) + } + } + is RealmToMigrate.DynamicRealm -> { + val pickledSessions = mutableListOf() + realm.schema.get("OlmSessionEntity")?.transform { + val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA) + val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY) + val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS) + val olmSession = deserializeFromRealm(sessionData)!! + val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString() + val pickledSession = PickledSession( + pickle = pickle, + senderKey = deviceKey, + createdUsingFallbackKey = false, + creationTime = lastReceivedMessageTs.toString(), + lastUseTime = lastReceivedMessageTs.toString() + ) + // should we check the tracking status? + pickledSessions.add(pickledSession) + if (pickledSessions.size > chunkSize) { + onChunk(pickledSessions.toImmutableList()) + pickledSessions.clear() + } + } + if (pickledSessions.isNotEmpty()) { + onChunk(pickledSessions) + } + } + } +} + +private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession { + val deviceKey = this.deviceKey ?: "" + val lastReceivedMessageTs = this.lastReceivedMessageTs + val olmSessionStr = this.olmSessionData + val olmSession = deserializeFromRealm(olmSessionStr)!! + val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString() + return PickledSession( + pickle = pickledOlmSession, + senderKey = deviceKey, + createdUsingFallbackKey = false, + creationTime = lastReceivedMessageTs.toString(), + lastUseTime = lastReceivedMessageTs.toString() + ) +} + +private val charset = Charset.forName("UTF-8") +private fun ByteArray.asString() = String(this, charset) diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt index 741b5a4c8fe..5bfafb44e88 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt @@ -16,9 +16,11 @@ package org.matrix.android.sdk.internal.session +import io.realm.DynamicRealm import io.realm.Realm import io.realm.RealmConfiguration import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase +import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate import org.matrix.rustcomponents.sdk.crypto.ProgressListener import timber.log.Timber import java.io.File @@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) Timber.v("OnProgress: $progress/$total") } } - Realm.getInstance(cryptoRealm).use { realm -> - extractMigrationData.extractData(realm) { + extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) { org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener) } } @@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) } return rustFilesDir } + + fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) { + if (!rustFilesDir.exists()) { + rustFilesDir.mkdir() + } + val extractMigrationData = ExtractMigrationDataUseCase() + + try { + val progressListener = object : ProgressListener { + override fun onProgress(progress: Int, total: Int) { + Timber.v("OnProgress: $progress/$total") + } + } + extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) { + org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener) + } + } catch (failure: Throwable) { + Timber.e(failure, "Failure while calling rust migration method") + throw failure + } + } } From fe171fa057f8e5552be90c48b3d6798ae276a946 Mon Sep 17 00:00:00 2001 From: valere Date: Sat, 6 May 2023 10:00:42 +0200 Subject: [PATCH 4/9] update changelog --- changelog.d/8405.sdk | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8405.sdk diff --git a/changelog.d/8405.sdk b/changelog.d/8405.sdk new file mode 100644 index 00000000000..c8d165fcdc8 --- /dev/null +++ b/changelog.d/8405.sdk @@ -0,0 +1 @@ +Add crypto database migration 22, that extract account and olm session to the new rust DB format From 174040688fcf0fbde625811064568a6f98fafc73 Mon Sep 17 00:00:00 2001 From: valere Date: Sat, 6 May 2023 10:01:56 +0200 Subject: [PATCH 5/9] fix changelog --- .../internal/crypto/store/db/migration/MigrateCryptoTo022.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt index d26fe017411..2c9405b1502 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 New Vector Ltd + * Copyright 2023 The Matrix.org Foundation C.I.C. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From e4d494e2a25b142e070aa5d0633064c335ed5dc4 Mon Sep 17 00:00:00 2001 From: valere Date: Sun, 7 May 2023 16:20:17 +0200 Subject: [PATCH 6/9] Fix migration test per source set --- .../database/CryptoSanityMigrationTest.kt | 69 +++++++++++++++++++ .../database/CryptoSanityMigrationTest.kt | 7 +- 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt rename matrix-sdk-android/src/{androidTest => androidTestRustCrypto}/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt (93%) diff --git a/matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt new file mode 100644 index 00000000000..b4f07eff5a4 --- /dev/null +++ b/matrix-sdk-android/src/androidTestKotlinCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2022 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.database + +import android.content.Context +import androidx.test.platform.app.InstrumentationRegistry +import io.realm.Realm +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration +import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule +import org.matrix.android.sdk.internal.util.time.Clock + +class CryptoSanityMigrationTest { + @get:Rule val configurationFactory = TestRealmConfigurationFactory() + + lateinit var context: Context + var realm: Realm? = null + + @Before + fun setUp() { + context = InstrumentationRegistry.getInstrumentation().context + } + + @After + fun tearDown() { + realm?.close() + } + + @Test + fun cryptoDatabaseShouldMigrateGracefully() { + val realmName = "crypto_store_20.realm" + + val migration = RealmCryptoStoreMigration( + object : Clock { + override fun epochMillis(): Long { + return 0L + } + } + ) + + val realmConfiguration = configurationFactory.createConfiguration( + realmName, + "7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca", + RealmCryptoStoreModule(), + migration.schemaVersion, + migration + ) + configurationFactory.copyRealmFromAssets(context, realmName, realmName) + + realm = Realm.getInstance(realmConfiguration) + } +} diff --git a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt similarity index 93% rename from matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt rename to matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt index 762129a3f70..9f97640e46c 100644 --- a/matrix-sdk-android/src/androidTest/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2022 The Matrix.org Foundation C.I.C. + * Copyright 2023 The Matrix.org Foundation C.I.C. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule import org.matrix.android.sdk.internal.util.time.Clock +import org.matrix.olm.OlmManager import java.io.File import java.security.KeyStore @@ -41,6 +42,8 @@ class CryptoSanityMigrationTest { @Before fun setUp() { + // Ensure Olm is initialized + OlmManager() context = InstrumentationRegistry.getInstrumentation().context } @@ -68,7 +71,7 @@ class CryptoSanityMigrationTest { SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) ) ), - File("test_rust") + File(configurationFactory.root, "rust-sdk") ) val realmConfiguration = configurationFactory.createConfiguration( From 5e2863111d5e397d3f2c56cef043185ee7f8dd34 Mon Sep 17 00:00:00 2001 From: valere Date: Tue, 9 May 2023 09:26:47 +0200 Subject: [PATCH 7/9] post rebase fix --- ...icElementAndroidToElementRMigrationTest.kt | 50 ++++------- .../database/CryptoSanityMigrationTest.kt | 18 ++-- .../store/db/RealmCryptoStoreMigration.kt | 14 +-- .../store/db/RustMigrationInfoProvider.kt | 31 +++++++ .../store/db/migration/MigrateCryptoTo022.kt | 5 +- .../rust/ExtractMigrationDataFailure.kt | 3 +- .../rust/ExtractMigrationDataUseCase.kt | 86 +++---------------- .../store/db/migration/rust/ExtractUtils.kt | 82 ++++++++++++++++++ .../session/MigrateEAtoEROperation.kt | 2 +- 9 files changed, 165 insertions(+), 126 deletions(-) create mode 100644 matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt index e6f027d4083..f7439444eb7 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt @@ -27,20 +27,20 @@ import org.junit.After import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Before -import org.junit.Ignore import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith -import org.matrix.android.sdk.InstrumentedTest import org.matrix.android.sdk.TestBuildVersionSdkIntProvider import org.matrix.android.sdk.api.securestorage.SecretStoringUtils import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule +import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity import org.matrix.android.sdk.internal.database.RealmKeysUtils import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory import org.matrix.android.sdk.internal.util.time.Clock +import org.matrix.android.sdk.test.shared.createTimberTestRule import org.matrix.olm.OlmAccount import org.matrix.olm.OlmManager import org.matrix.rustcomponents.sdk.crypto.OlmMachine @@ -48,10 +48,13 @@ import java.io.File import java.security.KeyStore @RunWith(AndroidJUnit4::class) -class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest { +class DynamicElementAndroidToElementRMigrationTest { @get:Rule val configurationFactory = TestRealmConfigurationFactory() + @Rule + fun timberTestRule() = createTimberTestRule() + var context: Context = InstrumentationRegistry.getInstrumentation().context var realm: Realm? = null @@ -81,25 +84,26 @@ class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest { } @Test -<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() { testMigrate(false) } @Test - @Ignore("We don't migrate group session for now, and it makes test suite unstable") fun given_a_valid_crypto_store_realm_file_no_lazy_then_migration_should_be_successful() { testMigrate(true) } private fun testMigrate(migrateGroupSessions: Boolean) { -======= - fun dynamic_migration_to_rust() { val targetFile = File(configurationFactory.root, "rust-sdk") ->>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt val realmName = "crypto_store_migration_16.realm" - val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile) + val infoProvider = RustMigrationInfoProvider( + targetFile, + rustEncryptionConfiguration + ).apply { + migrateMegolmGroupSessions = migrateGroupSessions + } + val migration = RealmCryptoStoreMigration(fakeClock, infoProvider) val realmConfiguration = configurationFactory.createConfiguration( realmName, @@ -116,17 +120,7 @@ class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest { val deviceId = metaData.deviceId!! val olmAccount = metaData.getOlmAccount()!! -<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt - val extractor = MigrateEAtoEROperation(migrateGroupSessions) - - val targetFile = File(configurationFactory.root, "rust-sdk") - - extractor.execute(realmConfiguration, targetFile, null) - - val machine = OlmMachine(userId, deviceId, targetFile.path, null) -======= val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase()) ->>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"]) assertNotNull(machine.getBackupKeys()) @@ -135,20 +129,12 @@ class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest { assertTrue(crossSigningStatus.hasSelfSigning) assertTrue(crossSigningStatus.hasUserSigning) -<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt if (migrateGroupSessions) { - val inboundGroupSessionEntities = realm!!.where().findAll() - assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt()) - - val backedUpInboundGroupSessionEntities = realm!! - .where() - .equalTo(OlmInboundGroupSessionEntityFields.BACKED_UP, true) - .findAll() - assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt()) + assertTrue("Some outbound sessions should be migrated", machine.roomKeyCounts().total.toInt() > 0) + assertTrue("There are some backed-up sessions", machine.roomKeyCounts().backedUp.toInt() > 0) + } else { + assertTrue(machine.roomKeyCounts().total.toInt() == 0) + assertTrue(machine.roomKeyCounts().backedUp.toInt() == 0) } -======= - // How to check that olm sessions have been migrated? - // Can see it from logs ->>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt } } diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt index 9f97640e46c..828c0f51d4e 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/database/CryptoSanityMigrationTest.kt @@ -29,6 +29,7 @@ import org.matrix.android.sdk.api.securestorage.SecretStoringUtils import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule +import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider import org.matrix.android.sdk.internal.util.time.Clock import org.matrix.olm.OlmManager import java.io.File @@ -58,12 +59,8 @@ class CryptoSanityMigrationTest { fun cryptoDatabaseShouldMigrateGracefully() { val realmName = "crypto_store_20.realm" - val migration = RealmCryptoStoreMigration( - object : Clock { - override fun epochMillis(): Long { - return 0L - } - }, + val rustMigrationInfo = RustMigrationInfoProvider( + File(configurationFactory.root, "test_rust"), RustEncryptionConfiguration( "foo", RealmKeysUtils( @@ -71,7 +68,14 @@ class CryptoSanityMigrationTest { SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false) ) ), - File(configurationFactory.root, "rust-sdk") + ) + val migration = RealmCryptoStoreMigration( + object : Clock { + override fun epochMillis(): Long { + return 0L + } + }, + rustMigrationInfo ) val realmConfiguration = configurationFactory.createConfiguration( diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt index 76054a52578..99734f654fc 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RealmCryptoStoreMigration.kt @@ -17,7 +17,6 @@ package org.matrix.android.sdk.internal.crypto.store.db import io.realm.DynamicRealm -import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX @@ -40,10 +39,8 @@ import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020 import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021 import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022 -import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration import org.matrix.android.sdk.internal.util.time.Clock -import java.io.File import javax.inject.Inject /** @@ -54,9 +51,7 @@ import javax.inject.Inject */ internal class RealmCryptoStoreMigration @Inject constructor( private val clock: Clock, - private val rustEncryptionConfiguration: RustEncryptionConfiguration, - @SessionRustFilesDirectory - private val rustDirectory: File, + private val rustMigrationInfoProvider: RustMigrationInfoProvider, ) : MatrixRealmMigration( dbName = "Crypto", schemaVersion = 22L, @@ -90,6 +85,11 @@ internal class RealmCryptoStoreMigration @Inject constructor( if (oldVersion < 19) MigrateCryptoTo019(realm).perform() if (oldVersion < 20) MigrateCryptoTo020(realm).perform() if (oldVersion < 21) MigrateCryptoTo021(realm).perform() - if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform() + if (oldVersion < 22) MigrateCryptoTo022( + realm, + rustMigrationInfoProvider.rustDirectory, + rustMigrationInfoProvider.rustEncryptionConfiguration, + rustMigrationInfoProvider.migrateMegolmGroupSessions + ).perform() } } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt new file mode 100644 index 00000000000..667990468c1 --- /dev/null +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/RustMigrationInfoProvider.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2023 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.crypto.store.db + +import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration +import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory +import java.io.File +import javax.inject.Inject + +internal class RustMigrationInfoProvider @Inject constructor( + @SessionRustFilesDirectory + val rustDirectory: File, + val rustEncryptionConfiguration: RustEncryptionConfiguration +) { + + var migrateMegolmGroupSessions: Boolean = false +} diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt index 2c9405b1502..d0f612aa87d 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/MigrateCryptoTo022.kt @@ -28,14 +28,15 @@ import java.io.File internal class MigrateCryptoTo022( realm: DynamicRealm, private val rustDirectory: File, - private val rustEncryptionConfiguration: RustEncryptionConfiguration + private val rustEncryptionConfiguration: RustEncryptionConfiguration, + private val migrateMegolmGroupSessions: Boolean = false ) : RealmMigrator( realm, 22 ) { override fun doMigrate(realm: DynamicRealm) { // Migrate to rust! - val migrateOperation = MigrateEAtoEROperation() + val migrateOperation = MigrateEAtoEROperation(migrateMegolmGroupSessions) migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase()) // wa can't delete all for now, but we can do some cleaning diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt index 8e864029163..fb4bd1c8fe0 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataFailure.kt @@ -16,4 +16,5 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust -object ExtractMigrationDataFailure : java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.") +data class ExtractMigrationDataFailure(override val cause: Throwable) : + java.lang.RuntimeException("Can't proceed with migration, crypto store is empty or some necessary data is missing.", cause) diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt index 5431f17dae0..3dae9a6b13a 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractMigrationDataUseCase.kt @@ -19,29 +19,19 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust import io.realm.Realm import io.realm.RealmConfiguration import io.realm.kotlin.where -import org.matrix.android.sdk.api.extensions.orFalse -import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity -import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity -import org.matrix.olm.OlmSession import org.matrix.olm.OlmUtility import org.matrix.rustcomponents.sdk.crypto.MigrationData -import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession -import org.matrix.rustcomponents.sdk.crypto.PickledSession import timber.log.Timber -import java.nio.charset.Charset import kotlin.system.measureTimeMillis -private val charset = Charset.forName("UTF-8") - -internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) { +internal class ExtractMigrationDataUseCase(private val migrateGroupSessions: Boolean = false) { fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) { return try { extract(realm, importPartial) } catch (failure: Throwable) { - throw ExtractMigrationDataFailure + throw ExtractMigrationDataFailure(failure) } } @@ -88,75 +78,19 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f // We are going to do it lazyly when decryption fails if (migrateGroupSessions) { var migratedInboundGroupSessionCount = 0 - readTime = 0 - writeTime = 0 measureTimeMillis { - realm.where() - .findAll() - .chunked(chunkSize) { chunk -> - val export: List - measureTimeMillis { - export = chunk.mapNotNull { it.toPickledInboundGroupSession(pickleKey) } - }.also { - readTime += it - } - migratedInboundGroupSessionCount += export.size - measureTimeMillis { - importPartial( - baseExtract.copy(inboundGroupSessions = export) - ) - }.also { - writeTime += it - } - } + realm.pickledOlmGroupSessions(pickleKey, chunkSize) { pickledSessions -> + migratedInboundGroupSessionCount += pickledSessions.size + measureTimeMillis { + importPartial( + baseExtract.copy(inboundGroupSessions = pickledSessions) + ) + }.also { writeTime += it } + } }.also { Timber.i("Migration: took $it ms to migrate $migratedInboundGroupSessionCount group sessions") - Timber.i("Migration: extract time $readTime") Timber.i("Migration: rust import time $writeTime") } } - -// return baseExtract - } - - private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession? { - val senderKey = this.senderKey ?: return null - val backedUp = this.backedUp - val olmInboundGroupSession = this.getOlmGroupSession() ?: return null.also { - Timber.w("Rust db migration: Failed to migrated group session $sessionId") - } - val data = this.getData() ?: return null.also { - Timber.w("Rust db migration: Failed to migrated group session $sessionId, no meta data") - } - val roomId = data.roomId ?: return null.also { - Timber.w("Rust db migration: Failed to migrated group session $sessionId, no roomId") - } - val pickledInboundGroupSession = olmInboundGroupSession.pickle(pickleKey, StringBuffer()).asString() - return PickledInboundGroupSession( - pickle = pickledInboundGroupSession, - senderKey = senderKey, - signingKey = data.keysClaimed.orEmpty(), - roomId = roomId, - forwardingChains = data.forwardingCurve25519KeyChain.orEmpty(), - imported = data.trusted.orFalse().not(), - backedUp = backedUp - ) } - - private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession { - val deviceKey = this.deviceKey ?: "" - val lastReceivedMessageTs = this.lastReceivedMessageTs - val olmSessionStr = this.olmSessionData - val olmSession = deserializeFromRealm(olmSessionStr)!! - val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString() - return PickledSession( - pickle = pickledOlmSession, - senderKey = deviceKey, - createdUsingFallbackKey = false, - creationTime = lastReceivedMessageTs.toString(), - lastUseTime = lastReceivedMessageTs.toString() - ) - } - - private fun ByteArray.asString() = String(this, charset) } diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt index 343960b92bd..608f68fc3dd 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/crypto/store/db/migration/rust/ExtractUtils.kt @@ -18,19 +18,27 @@ package org.matrix.android.sdk.internal.crypto.store.db.migration.rust import io.realm.kotlin.where import okhttp3.internal.toImmutableList +import org.matrix.android.sdk.api.extensions.orFalse +import org.matrix.android.sdk.internal.crypto.model.InboundGroupSessionData import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields +import org.matrix.android.sdk.internal.di.MoshiProvider import org.matrix.olm.OlmAccount +import org.matrix.olm.OlmInboundGroupSession import org.matrix.olm.OlmSession import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport import org.matrix.rustcomponents.sdk.crypto.MigrationData import org.matrix.rustcomponents.sdk.crypto.PickledAccount +import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession import org.matrix.rustcomponents.sdk.crypto.PickledSession +import timber.log.Timber import java.nio.charset.Charset sealed class RealmToMigrate { @@ -225,6 +233,80 @@ fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onCh } } +private val sessionDataAdapter = MoshiProvider.providesMoshi() + .adapter(InboundGroupSessionData::class.java) +fun RealmToMigrate.pickledOlmGroupSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List) -> Unit)) { + when (this) { + is RealmToMigrate.ClassicRealm -> { + realm.where() + .findAll() + .chunked(chunkSize) { chunk -> + val export = chunk.mapNotNull { it.toPickledInboundGroupSession(pickleKey) } + onChunk(export) + } + } + is RealmToMigrate.DynamicRealm -> { + val pickledSessions = mutableListOf() + realm.schema.get("OlmInboundGroupSessionEntity")?.transform { + val senderKey = it.getString(OlmInboundGroupSessionEntityFields.SENDER_KEY) + val roomId = it.getString(OlmInboundGroupSessionEntityFields.ROOM_ID) + val backedUp = it.getBoolean(OlmInboundGroupSessionEntityFields.BACKED_UP) + val serializedOlmInboundGroupSession = it.getString(OlmInboundGroupSessionEntityFields.SERIALIZED_OLM_INBOUND_GROUP_SESSION) + val inboundSession = deserializeFromRealm(serializedOlmInboundGroupSession) ?: return@transform Unit.also { + Timber.w("Rust db migration: Failed to migrated group session, no meta data") + } + val sessionData = it.getString(OlmInboundGroupSessionEntityFields.INBOUND_GROUP_SESSION_DATA_JSON).let { json -> + sessionDataAdapter.fromJson(json) + } ?: return@transform Unit.also { + Timber.w("Rust db migration: Failed to migrated group session, no meta data") + } + val pickle = inboundSession.pickle(pickleKey, StringBuffer()).asString() + val pickledSession = PickledInboundGroupSession( + pickle = pickle, + senderKey = senderKey, + signingKey = sessionData.keysClaimed.orEmpty(), + roomId = roomId, + forwardingChains = sessionData.forwardingCurve25519KeyChain.orEmpty(), + imported = sessionData.trusted.orFalse().not(), + backedUp = backedUp + ) + // should we check the tracking status? + pickledSessions.add(pickledSession) + if (pickledSessions.size > chunkSize) { + onChunk(pickledSessions.toImmutableList()) + pickledSessions.clear() + } + } + if (pickledSessions.isNotEmpty()) { + onChunk(pickledSessions) + } + } + } +} + +private fun OlmInboundGroupSessionEntity.toPickledInboundGroupSession(pickleKey: ByteArray): PickledInboundGroupSession? { + val senderKey = this.senderKey ?: return null + val backedUp = this.backedUp + val olmInboundGroupSession = this.getOlmGroupSession() ?: return null.also { + Timber.w("Rust db migration: Failed to migrated group session $sessionId") + } + val data = this.getData() ?: return null.also { + Timber.w("Rust db migration: Failed to migrated group session $sessionId, no meta data") + } + val roomId = data.roomId ?: return null.also { + Timber.w("Rust db migration: Failed to migrated group session $sessionId, no roomId") + } + val pickledInboundGroupSession = olmInboundGroupSession.pickle(pickleKey, StringBuffer()).asString() + return PickledInboundGroupSession( + pickle = pickledInboundGroupSession, + senderKey = senderKey, + signingKey = data.keysClaimed.orEmpty(), + roomId = roomId, + forwardingChains = data.forwardingCurve25519KeyChain.orEmpty(), + imported = data.trusted.orFalse().not(), + backedUp = backedUp + ) +} private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession { val deviceKey = this.deviceKey ?: "" val lastReceivedMessageTs = this.lastReceivedMessageTs diff --git a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt index 5bfafb44e88..b4944edbb91 100644 --- a/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt +++ b/matrix-sdk-android/src/rustCrypto/java/org/matrix/android/sdk/internal/session/MigrateEAtoEROperation.kt @@ -59,7 +59,7 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) if (!rustFilesDir.exists()) { rustFilesDir.mkdir() } - val extractMigrationData = ExtractMigrationDataUseCase() + val extractMigrationData = ExtractMigrationDataUseCase(migrateGroupSessions) try { val progressListener = object : ProgressListener { From f5a406bea67ca423d486ed83602311b1adee22f1 Mon Sep 17 00:00:00 2001 From: valere Date: Tue, 9 May 2023 10:28:48 +0200 Subject: [PATCH 8/9] test legacy is partially cleaned --- .../DynamicElementAndroidToElementRMigrationTest.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt index f7439444eb7..77dcb6b477e 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt @@ -37,6 +37,7 @@ import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule import org.matrix.android.sdk.internal.crypto.store.db.RustMigrationInfoProvider import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity +import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity import org.matrix.android.sdk.internal.database.RealmKeysUtils import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory import org.matrix.android.sdk.internal.util.time.Clock @@ -136,5 +137,9 @@ class DynamicElementAndroidToElementRMigrationTest { assertTrue(machine.roomKeyCounts().total.toInt() == 0) assertTrue(machine.roomKeyCounts().backedUp.toInt() == 0) } + + // legacy olm sessions should have been deleted + val remainingOlmSessions = realm!!.where().findAll().size + assertEquals("legacy olm sessions should have been removed from store",0, remainingOlmSessions) } } From d499b0721446cae4a81fa5fc73d1741935d99608 Mon Sep 17 00:00:00 2001 From: valere Date: Tue, 9 May 2023 11:16:58 +0200 Subject: [PATCH 9/9] Ignore a test for flackyness --- .../migration/DynamicElementAndroidToElementRMigrationTest.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt index 77dcb6b477e..52a75d0653c 100644 --- a/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt +++ b/matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt @@ -27,6 +27,7 @@ import org.junit.After import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Before +import org.junit.Ignore import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith @@ -90,6 +91,7 @@ class DynamicElementAndroidToElementRMigrationTest { } @Test + @Ignore("We don't migrate group sessions for now, and it's making this test suite unstable") fun given_a_valid_crypto_store_realm_file_no_lazy_then_migration_should_be_successful() { testMigrate(true) } @@ -140,6 +142,6 @@ class DynamicElementAndroidToElementRMigrationTest { // legacy olm sessions should have been deleted val remainingOlmSessions = realm!!.where().findAll().size - assertEquals("legacy olm sessions should have been removed from store",0, remainingOlmSessions) + assertEquals("legacy olm sessions should have been removed from store", 0, remainingOlmSessions) } }