Skip to content

Commit 06eff55

Browse files
Syer10Syer10
authored
Updater cleanup and improvements (#416)
1 parent 71730fd commit 06eff55

File tree

9 files changed

+100
-108
lines changed

9 files changed

+100
-108
lines changed

‎server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt

+10-13
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package suwayomi.tachidesk.manga.controller
22

33
import io.javalin.http.HttpCode
44
import io.javalin.websocket.WsConfig
5-
import kotlinx.coroutines.runBlocking
65
import mu.KotlinLogging
76
import org.kodein.di.DI
87
import org.kodein.di.conf.global
@@ -15,6 +14,7 @@ import suwayomi.tachidesk.manga.impl.update.UpdateStatus
1514
import suwayomi.tachidesk.manga.impl.update.UpdaterSocket
1615
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
1716
import suwayomi.tachidesk.manga.model.dataclass.MangaChapterDataClass
17+
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
1818
import suwayomi.tachidesk.manga.model.dataclass.PaginatedList
1919
import suwayomi.tachidesk.server.JavalinSetup.future
2020
import suwayomi.tachidesk.server.util.formParam
@@ -68,22 +68,18 @@ object UpdateController {
6868
}
6969
},
7070
behaviorOf = { ctx, categoryId ->
71-
val categoriesForUpdate = ArrayList<CategoryDataClass>()
7271
if (categoryId == null) {
7372
logger.info { "Adding Library to Update Queue" }
74-
categoriesForUpdate.addAll(Category.getCategoryList())
73+
addCategoriesToUpdateQueue(Category.getCategoryList(), true)
7574
} else {
7675
val category = Category.getCategoryById(categoryId)
7776
if (category != null) {
78-
categoriesForUpdate.add(category)
77+
addCategoriesToUpdateQueue(listOf(category), true)
7978
} else {
8079
logger.info { "No Category found" }
8180
ctx.status(HttpCode.BAD_REQUEST)
82-
return@handler
8381
}
8482
}
85-
addCategoriesToUpdateQueue(categoriesForUpdate, true)
86-
ctx.status(HttpCode.OK)
8783
},
8884
withResults = {
8985
httpCode(HttpCode.OK)
@@ -94,14 +90,15 @@ object UpdateController {
9490
private fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean = false) {
9591
val updater by DI.global.instance<IUpdater>()
9692
if (clear) {
97-
runBlocking { updater.reset() }
93+
updater.reset()
9894
}
99-
categories.forEach { category ->
100-
val mangas = CategoryManga.getCategoryMangaList(category.id)
101-
mangas.forEach { manga ->
95+
categories
96+
.flatMap { CategoryManga.getCategoryMangaList(it.id) }
97+
.distinctBy { it.id }
98+
.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title))
99+
.forEach { manga ->
102100
updater.addMangaToQueue(manga)
103101
}
104-
}
105102
}
106103

107104
fun categoryUpdateWS(ws: WsConfig) {
@@ -125,7 +122,7 @@ object UpdateController {
125122
},
126123
behaviorOf = { ctx ->
127124
val updater by DI.global.instance<IUpdater>()
128-
ctx.json(updater.getStatus().value.getJsonSummary())
125+
ctx.json(updater.status.value)
129126
},
130127
withResults = {
131128
json<UpdateStatus>(HttpCode.OK)

‎server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@ import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
55

66
interface IUpdater {
77
fun addMangaToQueue(manga: MangaDataClass)
8-
fun getStatus(): StateFlow<UpdateStatus>
9-
suspend fun reset(): Unit
8+
val status: StateFlow<UpdateStatus>
9+
fun reset()
1010
}

‎server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt

+4-6
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,7 @@ enum class JobStatus {
99
FAILED
1010
}
1111

12-
class UpdateJob(val manga: MangaDataClass, var status: JobStatus = JobStatus.PENDING) {
13-
14-
override fun toString(): String {
15-
return "UpdateJob(status=$status, manga=${manga.title})"
16-
}
17-
}
12+
data class UpdateJob(
13+
val manga: MangaDataClass,
14+
val status: JobStatus = JobStatus.PENDING
15+
)

‎server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt

+14-24
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,23 @@
11
package suwayomi.tachidesk.manga.impl.update
22

3+
import com.fasterxml.jackson.annotation.JsonIgnore
34
import mu.KotlinLogging
45
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
56

6-
var logger = KotlinLogging.logger {}
7-
class UpdateStatus(
8-
var statusMap: MutableMap<JobStatus, MutableList<MangaDataClass>> = mutableMapOf<JobStatus, MutableList<MangaDataClass>>(),
9-
var running: Boolean = false,
7+
val logger = KotlinLogging.logger {}
8+
data class UpdateStatus(
9+
val statusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
10+
val running: Boolean = false,
11+
@JsonIgnore
12+
val numberOfJobs: Int = 0
1013
) {
11-
var numberOfJobs: Int = 0
1214

1315
constructor(jobs: List<UpdateJob>, running: Boolean) : this(
14-
mutableMapOf<JobStatus, MutableList<MangaDataClass>>(),
15-
running
16-
) {
17-
this.numberOfJobs = jobs.size
18-
jobs.forEach {
19-
val list = statusMap.getOrDefault(it.status, mutableListOf())
20-
list.add(it.manga)
21-
statusMap[it.status] = list
22-
}
23-
}
24-
25-
override fun toString(): String {
26-
return "UpdateStatus(statusMap=${statusMap.map { "${it.key} : ${it.value.size}" }.joinToString("; ")}, running=$running)"
27-
}
28-
29-
// serialize to summary json
30-
fun getJsonSummary(): String {
31-
return """{"statusMap":{${statusMap.map { "\"${it.key}\" : ${it.value.size}" }.joinToString(",")}}, "running":$running}"""
32-
}
16+
statusMap = jobs.groupBy { it.status }
17+
.mapValues { entry ->
18+
entry.value.map { it.manga }
19+
},
20+
running = running,
21+
numberOfJobs = jobs.size
22+
)
3323
}

‎server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

+38-36
Original file line numberDiff line numberDiff line change
@@ -3,74 +3,76 @@ package suwayomi.tachidesk.manga.impl.update
33
import kotlinx.coroutines.CancellationException
44
import kotlinx.coroutines.CoroutineScope
55
import kotlinx.coroutines.Dispatchers
6-
import kotlinx.coroutines.Job
76
import kotlinx.coroutines.SupervisorJob
8-
import kotlinx.coroutines.cancel
7+
import kotlinx.coroutines.cancelChildren
98
import kotlinx.coroutines.channels.Channel
109
import kotlinx.coroutines.flow.MutableStateFlow
11-
import kotlinx.coroutines.flow.StateFlow
10+
import kotlinx.coroutines.flow.asStateFlow
11+
import kotlinx.coroutines.flow.catch
12+
import kotlinx.coroutines.flow.consumeAsFlow
13+
import kotlinx.coroutines.flow.launchIn
14+
import kotlinx.coroutines.flow.onEach
15+
import kotlinx.coroutines.flow.update
1216
import kotlinx.coroutines.launch
1317
import mu.KotlinLogging
1418
import suwayomi.tachidesk.manga.impl.Chapter
1519
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
20+
import java.util.concurrent.ConcurrentHashMap
1621

1722
class Updater : IUpdater {
1823
private val logger = KotlinLogging.logger {}
1924
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
2025

21-
private var tracker = mutableMapOf<String, UpdateJob>()
22-
private var updateChannel = Channel<UpdateJob>()
23-
private val statusChannel = MutableStateFlow(UpdateStatus())
24-
private var updateJob: Job? = null
26+
private val _status = MutableStateFlow(UpdateStatus())
27+
override val status = _status.asStateFlow()
2528

26-
init {
27-
updateJob = createUpdateJob()
28-
}
29+
private val tracker = ConcurrentHashMap<Int, UpdateJob>()
30+
private var updateChannel = createUpdateChannel()
2931

30-
private fun createUpdateJob(): Job {
31-
return scope.launch {
32-
while (true) {
33-
val job = updateChannel.receive()
34-
process(job)
35-
statusChannel.value = UpdateStatus(tracker.values.toList(), !updateChannel.isEmpty)
32+
private fun createUpdateChannel(): Channel<UpdateJob> {
33+
val channel = Channel<UpdateJob>(Channel.UNLIMITED)
34+
channel.consumeAsFlow()
35+
.onEach { job ->
36+
_status.value = UpdateStatus(
37+
process(job),
38+
tracker.any { (_, job) ->
39+
job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
40+
}
41+
)
3642
}
37-
}
43+
.catch { logger.error(it) { "Error during updates" } }
44+
.launchIn(scope)
45+
return channel
3846
}
3947

40-
private suspend fun process(job: UpdateJob) {
41-
job.status = JobStatus.RUNNING
42-
tracker["${job.manga.id}"] = job
43-
statusChannel.value = UpdateStatus(tracker.values.toList(), true)
44-
try {
48+
private suspend fun process(job: UpdateJob): List<UpdateJob> {
49+
tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING)
50+
_status.update { UpdateStatus(tracker.values.toList(), true) }
51+
tracker[job.manga.id] = try {
4552
logger.info { "Updating ${job.manga.title}" }
4653
Chapter.getChapterList(job.manga.id, true)
47-
job.status = JobStatus.COMPLETE
54+
job.copy(status = JobStatus.COMPLETE)
4855
} catch (e: Exception) {
4956
if (e is CancellationException) throw e
5057
logger.error(e) { "Error while updating ${job.manga.title}" }
51-
job.status = JobStatus.FAILED
58+
job.copy(status = JobStatus.FAILED)
5259
}
53-
tracker["${job.manga.id}"] = job
60+
return tracker.values.toList()
5461
}
5562

5663
override fun addMangaToQueue(manga: MangaDataClass) {
5764
scope.launch {
5865
updateChannel.send(UpdateJob(manga))
5966
}
60-
tracker["${manga.id}"] = UpdateJob(manga)
61-
statusChannel.value = UpdateStatus(tracker.values.toList(), true)
62-
}
63-
64-
override fun getStatus(): StateFlow<UpdateStatus> {
65-
return statusChannel
67+
tracker[manga.id] = UpdateJob(manga)
68+
_status.update { UpdateStatus(tracker.values.toList(), true) }
6669
}
6770

68-
override suspend fun reset() {
71+
override fun reset() {
72+
scope.coroutineContext.cancelChildren()
6973
tracker.clear()
74+
_status.update { UpdateStatus() }
7075
updateChannel.cancel()
71-
statusChannel.value = UpdateStatus()
72-
updateJob?.cancel("Reset")
73-
updateChannel = Channel()
74-
updateJob = createUpdateJob()
76+
updateChannel = createUpdateChannel()
7577
}
7678
}

‎server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdaterSocket.kt

+11-13
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,26 @@ import kotlinx.coroutines.CoroutineScope
66
import kotlinx.coroutines.Dispatchers
77
import kotlinx.coroutines.Job
88
import kotlinx.coroutines.SupervisorJob
9-
import kotlinx.coroutines.flow.collectLatest
10-
import kotlinx.coroutines.launch
9+
import kotlinx.coroutines.flow.launchIn
10+
import kotlinx.coroutines.flow.onEach
1111
import mu.KotlinLogging
1212
import org.kodein.di.DI
1313
import org.kodein.di.conf.global
1414
import org.kodein.di.instance
1515

16-
object UpdaterSocket : Websocket() {
16+
object UpdaterSocket : Websocket<UpdateStatus>() {
1717
private val logger = KotlinLogging.logger {}
1818
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
1919
private val updater by DI.global.instance<IUpdater>()
2020
private var job: Job? = null
2121

22-
override fun notifyClient(ctx: WsContext) {
23-
ctx.send(updater.getStatus().value.getJsonSummary())
22+
override fun notifyClient(ctx: WsContext, value: UpdateStatus?) {
23+
ctx.send(value ?: updater.status.value)
2424
}
2525

2626
override fun handleRequest(ctx: WsMessageContext) {
2727
when (ctx.message()) {
28-
"STATUS" -> notifyClient(ctx)
28+
"STATUS" -> notifyClient(ctx, updater.status.value)
2929
else -> ctx.send(
3030
"""
3131
|Invalid command.
@@ -40,7 +40,7 @@ object UpdaterSocket : Websocket() {
4040
override fun addClient(ctx: WsContext) {
4141
logger.info { ctx.sessionId }
4242
super.addClient(ctx)
43-
if (job == null) {
43+
if (job?.isActive != true) {
4444
job = start()
4545
}
4646
}
@@ -54,12 +54,10 @@ object UpdaterSocket : Websocket() {
5454
}
5555

5656
fun start(): Job {
57-
return scope.launch {
58-
while (true) {
59-
updater.getStatus().collectLatest {
60-
notifyAllClients()
61-
}
57+
return updater.status
58+
.onEach {
59+
notifyAllClients(it)
6260
}
63-
}
61+
.launchIn(scope)
6462
}
6563
}

‎server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Websocket.kt

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@ import io.javalin.websocket.WsContext
44
import io.javalin.websocket.WsMessageContext
55
import java.util.concurrent.ConcurrentHashMap
66

7-
abstract class Websocket {
7+
abstract class Websocket<T> {
88
protected val clients = ConcurrentHashMap<String, WsContext>()
99
open fun addClient(ctx: WsContext) {
1010
clients[ctx.sessionId] = ctx
11-
notifyClient(ctx)
11+
notifyClient(ctx, null)
1212
}
1313
open fun removeClient(ctx: WsContext) {
1414
clients.remove(ctx.sessionId)
1515
}
16-
open fun notifyAllClients() {
17-
clients.values.forEach { notifyClient(it) }
16+
open fun notifyAllClients(value: T) {
17+
clients.values.forEach { notifyClient(it, value) }
1818
}
19-
abstract fun notifyClient(ctx: WsContext)
19+
abstract fun notifyClient(ctx: WsContext, value: T?)
2020
abstract fun handleRequest(ctx: WsMessageContext)
2121
}

‎server/src/test/kotlin/suwayomi/tachidesk/manga/controller/UpdateControllerTest.kt

+3-3
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ internal class UpdateControllerTest : ApplicationTest() {
3232
UpdateController.categoryUpdate(ctx)
3333
verify { ctx.status(HttpCode.BAD_REQUEST) }
3434
val updater by DI.global.instance<IUpdater>()
35-
assertEquals(0, updater.getStatus().value.numberOfJobs)
35+
assertEquals(0, updater.status.value.numberOfJobs)
3636
}
3737

3838
@Test
@@ -44,7 +44,7 @@ internal class UpdateControllerTest : ApplicationTest() {
4444
UpdateController.categoryUpdate(ctx)
4545
verify { ctx.status(HttpCode.OK) }
4646
val updater by DI.global.instance<IUpdater>()
47-
assertEquals(1, updater.getStatus().value.numberOfJobs)
47+
assertEquals(1, updater.status.value.numberOfJobs)
4848
}
4949

5050
@Test
@@ -60,7 +60,7 @@ internal class UpdateControllerTest : ApplicationTest() {
6060
UpdateController.categoryUpdate(ctx)
6161
verify { ctx.status(HttpCode.OK) }
6262
val updater by DI.global.instance<IUpdater>()
63-
assertEquals(3, updater.getStatus().value.numberOfJobs)
63+
assertEquals(3, updater.status.value.numberOfJobs)
6464
}
6565

6666
private fun createLibraryManga(

0 commit comments

Comments
 (0)