Improve history import performance and fix some bugs (#5666)

* Improve history import performance and fix some bugs

* Make the view count optional

* Add lastViewedPlaylistItemId to the optional keys

* I forgot lastViewedPlaylistType

Author: absidue

src/constants.js

@@ -51,6 +51,7 @@ const DBActions = {
   },
 
   HISTORY: {
+    OVERWRITE: 'db-action-history-overwrite',
     UPDATE_WATCH_PROGRESS: 'db-action-history-update-watch-progress',
     UPDATE_PLAYLIST: 'db-action-history-update-playlist',
   },
@@ -78,6 +79,7 @@ const SyncEvents = {
   },
 
   HISTORY: {
+    OVERWRITE: 'sync-history-overwrite',
     UPDATE_WATCH_PROGRESS: 'sync-history-update-watch-progress',
     UPDATE_PLAYLIST: 'sync-history-update-playlist',
   },

src/datastores/handlers/base.js

@@ -56,6 +56,12 @@ class History {
     return db.history.updateAsync({ videoId: record.videoId }, record, { upsert: true })
   }
 
+  static async overwrite(records) {
+    await db.history.removeAsync({}, { multi: true })
+
+    await db.history.insertAsync(records)
+  }
+
   static updateWatchProgress(videoId, watchProgress) {
     return db.history.updateAsync({ videoId }, { $set: { watchProgress } }, { upsert: true })
   }

src/datastores/handlers/electron.js

@@ -32,6 +32,13 @@ class History {
     )
   }
 
+  static overwrite(records) {
+    return ipcRenderer.invoke(
+      IpcChannels.DB_HISTORY,
+      { action: DBActions.HISTORY.OVERWRITE, data: records }
+    )
+  }
+
   static updateWatchProgress(videoId, watchProgress) {
     return ipcRenderer.invoke(
       IpcChannels.DB_HISTORY,

src/datastores/handlers/web.js

@@ -29,6 +29,10 @@ class History {
     return baseHandlers.history.upsert(record)
   }
 
+  static overwrite(records) {
+    return baseHandlers.history.overwrite(records)
+  }
+
   static updateWatchProgress(videoId, watchProgress) {
     return baseHandlers.history.updateWatchProgress(videoId, watchProgress)
   }

src/main/index.js

@@ -1075,6 +1075,15 @@ function runApp() {
           )
           return null
 
+        case DBActions.HISTORY.OVERWRITE:
+          await baseHandlers.history.overwrite(data)
+          syncOtherWindows(
+            IpcChannels.SYNC_HISTORY,
+            event,
+            { event: SyncEvents.HISTORY.OVERWRITE, data }
+          )
+          return null
+
         case DBActions.HISTORY.UPDATE_WATCH_PROGRESS:
           await baseHandlers.history.updateWatchProgress(data.videoId, data.watchProgress)
           syncOtherWindows(

src/renderer/components/data-settings/data-settings.js

@@ -50,6 +50,9 @@ export default defineComponent({
     allPlaylists: function () {
       return this.$store.getters.getAllPlaylists
     },
+    historyCacheById: function () {
+      return this.$store.getters.getHistoryCacheById
+    },
     historyCacheSorted: function () {
       return this.$store.getters.getHistoryCacheSorted
     },
@@ -616,7 +619,7 @@ export default defineComponent({
       })
     },
 
-    importFreeTubeHistory(textDecode) {
+    async importFreeTubeHistory(textDecode) {
       textDecode.pop()
 
       const requiredKeys = [
@@ -630,20 +633,24 @@ export default defineComponent({
         'title',
         'type',
         'videoId',
-        'viewCount',
         'watchProgress',
       ]
 
       const optionalKeys = [
         // `_id` absent if marked as watched manually
         '_id',
         'lastViewedPlaylistId',
+        'lastViewedPlaylistItemId',
+        'lastViewedPlaylistType',
+        'viewCount',
       ]
 
       const ignoredKeys = [
         'paid',
       ]
 
+      const historyItems = new Map(Object.entries(this.historyCacheById))
+
       textDecode.forEach((history) => {
         const historyData = JSON.parse(history)
         // We would technically already be done by the time the data is parsed,
@@ -667,14 +674,16 @@ export default defineComponent({
           showToast(this.$t('Settings.Data Settings.History object has insufficient data, skipping item'))
           console.error('Missing Keys: ', missingKeys, historyData)
         } else {
-          this.updateHistory(historyObject)
+          historyItems.set(historyObject.videoId, historyObject)
         }
       })
 
+      await this.overwriteHistory(historyItems)
+
       showToast(this.$t('Settings.Data Settings.All watched history has been successfully imported'))
     },
 
-    importYouTubeHistory(historyData) {
+    async importYouTubeHistory(historyData) {
       const filterPredicate = item =>
         item.products.includes('YouTube') &&
         item.titleUrl != null && // removed video doesnt contain url...
@@ -722,6 +731,8 @@ export default defineComponent({
         'activityControls',
       ].concat(Object.keys(keyMapping))
 
+      const historyItems = new Map(Object.entries(this.historyCacheById))
+
       filteredHistoryData.forEach(element => {
         const historyObject = {}
 
@@ -750,10 +761,12 @@ export default defineComponent({
           historyObject.watchProgress = 1
           historyObject.isLive = false
 
-          this.updateHistory(historyObject)
+          historyItems.set(historyObject.videoId, historyObject)
         }
       })
 
+      await this.overwriteHistory(historyItems)
+
       showToast(this.$t('Settings.Data Settings.All watched history has been successfully imported'))
     },
 
@@ -1069,10 +1082,10 @@ export default defineComponent({
     ...mapActions([
       'updateProfile',
       'updateShowProgressBar',
-      'updateHistory',
       'addPlaylist',
       'addVideo',
       'updatePlaylist',
+      'overwriteHistory'
     ]),
 
     ...mapMutations([

src/renderer/store/modules/history.js

@@ -45,6 +45,27 @@ const actions = {
     }
   },
 
+  /**
+   * @param {any} param0
+   * @param {Map<string, any>} historyItems
+   */
+  async overwriteHistory({ commit }, historyItems) {
+    try {
+      const sortedRecords = Array.from(historyItems.values())
+
+      // sort before sending saving to the database and passing to other windows
+      // so that the other windows can use it as is, without having to sort the array themselves
+      sortedRecords.sort((a, b) => b.timeWatched - a.timeWatched)
+
+      await DBHistoryHandlers.overwrite(sortedRecords)
+
+      commit('setHistoryCacheSorted', sortedRecords)
+      commit('setHistoryCacheById', Object.fromEntries(historyItems))
+    } catch (errMessage) {
+      console.error(errMessage)
+    }
+  },
+
   async removeFromHistory({ commit }, videoId) {
     try {
       await DBHistoryHandlers.delete(videoId)

src/renderer/store/modules/settings.js

@@ -476,6 +476,18 @@ const customActions = {
             commit('upsertToHistoryCache', data)
             break
 
+          case SyncEvents.HISTORY.OVERWRITE: {
+            const byId = {}
+            data.forEach(video => {
+              byId[video.videoId] = video
+            })
+
+            // It comes pre-sorted, so we don't have to sort it here
+            commit('setHistoryCacheSorted', data)
+            commit('setHistoryCacheById', byId)
+            break
+          }
+
           case SyncEvents.HISTORY.UPDATE_WATCH_PROGRESS:
             commit('updateRecordWatchProgressInHistoryCache', data)
             break