Downloader Rewrite (#437)

Syer10 · web-flow · commit 2195c3df765c · 2022-11-08T04:39:26.000+03:30
* Downloader rewrite
- Rewrite downloader to use coroutines instead of a thread
- Remove unused Page functions
- Add page progress
- Add ProgressResponseBody
- Add support for canceling a download in the middle of downloading
- Fix clear download queue

* Minor fix

* Minor improvements
- notifyAllClients now launches in another thread and only sends new data every second
- Better handling of download queue checker in step()
- Minor improvements and fixes

* Reorder downloads

* Download in parallel by source

* Remove TODO
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/OkHttpExtensions.kt
@@ -116,13 +116,13 @@ fun Call.asObservableSuccess(): Observable<Response> {
 @Suppress("UNUSED_PARAMETER")
 fun OkHttpClient.newCallWithProgress(request: Request, listener: ProgressListener): Call {
     val progressClient = newBuilder()
-//        .cache(null)
-//        .addNetworkInterceptor { chain ->
-//            val originalResponse = chain.proceed(chain.request())
-//            originalResponse.newBuilder()
-//                .body(ProgressResponseBody(originalResponse.body!!, listener))
-//                .build()
-//        }
+        .cache(null)
+        .addNetworkInterceptor { chain ->
+            val originalResponse = chain.proceed(chain.request())
+            originalResponse.newBuilder()
+                .body(ProgressResponseBody(originalResponse.body!!, listener))
+                .build()
+        }
         .build()
 
     return progressClient.newCall(request)
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/ProgressResponseBody.kt
@@ -0,0 +1,44 @@
+package eu.kanade.tachiyomi.network
+
+import okhttp3.MediaType
+import okhttp3.ResponseBody
+import okio.Buffer
+import okio.BufferedSource
+import okio.ForwardingSource
+import okio.Source
+import okio.buffer
+import java.io.IOException
+
+class ProgressResponseBody(private val responseBody: ResponseBody, private val progressListener: ProgressListener) : ResponseBody() {
+
+    private val bufferedSource: BufferedSource by lazy {
+        source(responseBody.source()).buffer()
+    }
+
+    override fun contentType(): MediaType? {
+        return responseBody.contentType()
+    }
+
+    override fun contentLength(): Long {
+        return responseBody.contentLength()
+    }
+
+    override fun source(): BufferedSource {
+        return bufferedSource
+    }
+
+    private fun source(source: Source): Source {
+        return object : ForwardingSource(source) {
+            var totalBytesRead = 0L
+
+            @Throws(IOException::class)
+            override fun read(sink: Buffer, byteCount: Long): Long {
+                val bytesRead = super.read(sink, byteCount)
+                // read() returns the number of bytes read, or -1 if this source is exhausted.
+                totalBytesRead += if (bytesRead != -1L) bytesRead else 0
+                progressListener.update(totalBytesRead, responseBody.contentLength(), bytesRead == -1L)
+                return bytesRead
+            }
+        }
+    }
+}
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/EpubPageLoader.kt
@@ -1,6 +1,5 @@
 package eu.kanade.tachiyomi.source.local.loader
 
-import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.util.storage.EpubFile
 import java.io.File
 
@@ -24,7 +23,6 @@ class EpubPageLoader(file: File) : PageLoader {
                 val streamFn = { epub.getInputStream(epub.getEntry(path)!!) }
                 ReaderPage(i).apply {
                     stream = streamFn
-                    status = Page.READY
                 }
             }
     }
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/RarPageLoader.kt
@@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.source.local.loader
 
 import com.github.junrar.Archive
 import com.github.junrar.rarfile.FileHeader
-import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import java.io.ByteArrayInputStream
@@ -46,7 +45,6 @@ class RarPageLoader(file: File) : PageLoader {
 
                     ReaderPage(i).apply {
                         stream = streamFn
-                        status = Page.READY
                     }
                 }
         }
@@ -58,7 +56,6 @@ class RarPageLoader(file: File) : PageLoader {
 
                 ReaderPage(i).apply {
                     stream = streamFn
-                    status = Page.READY
                 }
             }
     }
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/local/loader/ZipPageLoader.kt
@@ -1,6 +1,5 @@
 package eu.kanade.tachiyomi.source.local.loader
 
-import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder
 import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil
 import java.io.File
@@ -24,7 +23,6 @@ class ZipPageLoader(file: File) : PageLoader {
                 val streamFn = { zip.getInputStream(entry) }
                 ReaderPage(i).apply {
                     stream = streamFn
-                    status = Page.READY
                 }
             }
     }
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/model/Page.kt
@@ -2,7 +2,8 @@ package eu.kanade.tachiyomi.source.model
 
 import android.net.Uri
 import eu.kanade.tachiyomi.network.ProgressListener
-import rx.subjects.Subject
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.asStateFlow
 
 open class Page(
     val index: Int,
@@ -11,48 +12,17 @@ open class Page(
     @Transient var uri: Uri? = null // Deprecated but can't be deleted due to extensions
 ) : ProgressListener {
 
-    val number: Int
-        get() = index + 1
-
-    @Transient
-    @Volatile
-    var status: Int = 0
-        set(value) {
-            field = value
-            statusSubject?.onNext(value)
-            statusCallback?.invoke(this)
-        }
-
-    @Transient
-    @Volatile
-    var progress: Int = 0
-        set(value) {
-            field = value
-            statusCallback?.invoke(this)
-        }
-
-    @Transient
-    private var statusSubject: Subject<Int, Int>? = null
-
-    @Transient
-    private var statusCallback: ((Page) -> Unit)? = null
+    private val _progress = MutableStateFlow(0)
+    val progress = _progress.asStateFlow()
 
     override fun update(bytesRead: Long, contentLength: Long, done: Boolean) {
-        progress = if (contentLength > 0) {
+        _progress.value = if (contentLength > 0) {
             (100 * bytesRead / contentLength).toInt()
         } else {
             -1
         }
     }
 
-    fun setStatusSubject(subject: Subject<Int, Int>?) {
-        this.statusSubject = subject
-    }
-
-    fun setStatusCallback(f: ((Page) -> Unit)?) {
-        statusCallback = f
-    }
-
     companion object {
         const val QUEUE = 0
         const val LOAD_PAGE = 1
diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSourceFetcher.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/source/online/HttpSourceFetcher.kt
@@ -4,9 +4,7 @@ import eu.kanade.tachiyomi.source.model.Page
 import rx.Observable
 
 fun HttpSource.getImageUrl(page: Page): Observable<Page> {
-    page.status = Page.LOAD_PAGE
     return fetchImageUrl(page)
-        .doOnError { page.status = Page.ERROR }
         .onErrorReturn { null }
         .doOnNext { page.imageUrl = it }
         .map { page }
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/MangaAPI.kt
@@ -106,12 +106,13 @@ object MangaAPI {
 
             get("start", DownloadController.start)
             get("stop", DownloadController.stop)
-            get("clear", DownloadController.stop)
+            get("clear", DownloadController.clear)
         }
 
         path("download") {
             get("{mangaId}/chapter/{chapterIndex}", DownloadController.queueChapter)
             delete("{mangaId}/chapter/{chapterIndex}", DownloadController.unqueueChapter)
+            patch("{mangaId}/chapter/{chapterIndex}/reorder/{to}", DownloadController.reorderChapter)
             post("batch", DownloadController.queueChapters)
         }
 
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/DownloadController.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/controller/DownloadController.kt
@@ -46,10 +46,8 @@ object DownloadController {
                 description("Start the downloader")
             }
         },
-        behaviorOf = { ctx ->
+        behaviorOf = {
             DownloadManager.start()
-
-            ctx.status(200)
         },
         withResults = {
             httpCode(HttpCode.OK)
@@ -65,9 +63,9 @@ object DownloadController {
             }
         },
         behaviorOf = { ctx ->
-            DownloadManager.stop()
-
-            ctx.status(200)
+            ctx.future(
+                future { DownloadManager.stop() }
+            )
         },
         withResults = {
             httpCode(HttpCode.OK)
@@ -83,9 +81,9 @@ object DownloadController {
             }
         },
         behaviorOf = { ctx ->
-            DownloadManager.clear()
-
-            ctx.status(200)
+            ctx.future(
+                future { DownloadManager.clear() }
+            )
         },
         withResults = {
             httpCode(HttpCode.OK)
@@ -155,4 +153,23 @@ object DownloadController {
             httpCode(HttpCode.OK)
         }
     )
+
+    /** clear download queue */
+    val reorderChapter = handler(
+        pathParam<Int>("chapterIndex"),
+        pathParam<Int>("mangaId"),
+        pathParam<Int>("to"),
+        documentWith = {
+            withOperation {
+                summary("Downloader reorder chapter")
+                description("Reorder chapter in download queue")
+            }
+        },
+        behaviorOf = { _, chapterIndex, mangaId, to ->
+            DownloadManager.reorder(chapterIndex, mangaId, to)
+        },
+        withResults = {
+            httpCode(HttpCode.OK)
+        }
+    )
 }
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt
@@ -10,6 +10,7 @@ package suwayomi.tachidesk.manga.impl
 import eu.kanade.tachiyomi.source.local.LocalSource
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.online.HttpSource
+import kotlinx.coroutines.flow.StateFlow
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
@@ -37,7 +38,7 @@ object Page {
         return page.imageUrl!!
     }
 
-    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, useCache: Boolean = true): Pair<InputStream, String> {
+    suspend fun getPageImage(mangaId: Int, chapterIndex: Int, index: Int, useCache: Boolean = true, progressFlow: ((StateFlow<Int>) -> Unit)? = null): Pair<InputStream, String> {
         val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() }
         val source = getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
         val chapterEntry = transaction {
@@ -55,6 +56,7 @@ object Page {
             pageEntry[PageTable.url],
             pageEntry[PageTable.imageUrl]
         )
+        progressFlow?.invoke(tachiyomiPage.progress)
 
         // we treat Local source differently
         if (source.id == LocalSource.ID) {
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadManager.kt
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,5 @@`
`1`	`1`	`package eu.kanade.tachiyomi.source.local.loader`
`2`	`2`
`3`		`-import eu.kanade.tachiyomi.source.model.Page`
`4`	`3`	`import eu.kanade.tachiyomi.util.storage.EpubFile`
`5`	`4`	`import java.io.File`
`6`	`5`
`@@ -24,7 +23,6 @@ class EpubPageLoader(file: File) : PageLoader {`
`24`	`23`	`val streamFn = { epub.getInputStream(epub.getEntry(path)!!) }`
`25`	`24`	`ReaderPage(i).apply {`
`26`	`25`	`stream = streamFn`
`27`		`- status = Page.READY`
`28`	`26`	`}`
`29`	`27`	`}`
`30`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.source.local.loader`
`2`	`2`
`3`	`3`	`import com.github.junrar.Archive`
`4`	`4`	`import com.github.junrar.rarfile.FileHeader`
`5`		`-import eu.kanade.tachiyomi.source.model.Page`
`6`	`5`	`import eu.kanade.tachiyomi.util.lang.compareToCaseInsensitiveNaturalOrder`
`7`	`6`	`import suwayomi.tachidesk.manga.impl.util.storage.ImageUtil`
`8`	`7`	`import java.io.ByteArrayInputStream`
`@@ -46,7 +45,6 @@ class RarPageLoader(file: File) : PageLoader {`
`46`	`45`
`47`	`46`	`ReaderPage(i).apply {`
`48`	`47`	`stream = streamFn`
`49`		`- status = Page.READY`
`50`	`48`	`}`
`51`	`49`	`}`
`52`	`50`	`}`
`@@ -58,7 +56,6 @@ class RarPageLoader(file: File) : PageLoader {`
`58`	`56`
`59`	`57`	`ReaderPage(i).apply {`
`60`	`58`	`stream = streamFn`
`61`		`- status = Page.READY`
`62`	`59`	`}`
`63`	`60`	`}`
`64`	`61`	`}`
Original file line number	Diff line number	Diff line change
`@@ -106,12 +106,13 @@ object MangaAPI {`
`106`	`106`
`107`	`107`	`get("start", DownloadController.start)`
`108`	`108`	`get("stop", DownloadController.stop)`
`109`		`- get("clear", DownloadController.stop)`
	`109`	`+ get("clear", DownloadController.clear)`
`110`	`110`	`}`
`111`	`111`
`112`	`112`	`path("download") {`
`113`	`113`	`get("{mangaId}/chapter/{chapterIndex}", DownloadController.queueChapter)`
`114`	`114`	`delete("{mangaId}/chapter/{chapterIndex}", DownloadController.unqueueChapter)`
	`115`	`+ patch("{mangaId}/chapter/{chapterIndex}/reorder/{to}", DownloadController.reorderChapter)`
`115`	`116`	`post("batch", DownloadController.queueChapters)`
`116`	`117`	`}`
`117`	`118`