Skip to content

Commit 893ccca

Browse files
author
Jonas Kellerer
committed
feat: enable /aggregated to return the data as CSV
#217
1 parent cf3cc6e commit 893ccca

File tree

11 files changed

+401
-74
lines changed

11 files changed

+401
-74
lines changed

‎lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
package org.genspectrum.lapis.controller
22

3-
import com.fasterxml.jackson.databind.JsonNode
43
import org.apache.commons.csv.CSVFormat
54
import org.apache.commons.csv.CSVPrinter
6-
import org.genspectrum.lapis.silo.DetailsData
75
import org.springframework.stereotype.Component
86
import java.io.StringWriter
97

108
interface CsvRecord {
119
fun asArray(): Array<String>
10+
fun asHeader(): Array<String>
1211
}
1312

1413
@Component
@@ -31,12 +30,6 @@ class CsvWriter {
3130
}
3231
}
3332

34-
fun DetailsData.asCsvRecord() = JsonValuesCsvRecord(this.values)
35-
36-
data class JsonValuesCsvRecord(val values: Collection<JsonNode>) : CsvRecord {
37-
override fun asArray() = values.map { it.asText() }.toTypedArray()
38-
}
39-
4033
enum class Delimiter(val value: Char) {
4134
COMMA(','),
4235
TAB('\t'),

‎lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt

Lines changed: 169 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ import org.genspectrum.lapis.request.NucleotideMutation
1616
import org.genspectrum.lapis.request.OrderByField
1717
import org.genspectrum.lapis.request.SequenceFiltersRequestWithFields
1818
import org.genspectrum.lapis.response.AggregationData
19+
import org.genspectrum.lapis.response.DetailsData
1920
import org.genspectrum.lapis.response.MutationData
20-
import org.genspectrum.lapis.silo.DetailsData
2121
import org.springframework.http.MediaType
2222
import org.springframework.web.bind.annotation.GetMapping
2323
import org.springframework.web.bind.annotation.PostMapping
@@ -39,10 +39,16 @@ const val LIMIT_SCHEMA = "Limit"
3939
const val OFFSET_SCHEMA = "Offset"
4040

4141
const val DETAILS_ENDPOINT_DESCRIPTION = "Returns the specified metadata fields of sequences matching the filter."
42+
const val AGGREGATED_ENDPONT_DESCRIPTION = "Returns the number of sequences matching the specified sequence filters"
4243
const val AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION =
4344
"The fields to stratify by. If empty, only the overall count is returned"
45+
const val AGGREGATED_ORDER_BY_FIELDS_DESCRIPTION =
46+
"The fields of the response to order by." +
47+
"Fields specified here must either be \"count\" or also be present in \"fields\"."
4448
const val DETAILS_FIELDS_DESCRIPTION =
4549
"The fields that the response items should contain. If empty, all fields are returned"
50+
const val DETAILS_ORDER_BY_FIELDS_DESCRIPTION =
51+
"The fields of the response to order by. Fields specified here must also be present in \"fields\"."
4652
const val LIMIT_DESCRIPTION = "The maximum number of entries to return in the response"
4753
const val OFFSET_DESCRIPTION = "The offset of the first entry to return in the response. " +
4854
"This is useful for pagination in combination with \"limit\"."
@@ -53,7 +59,7 @@ class LapisController(
5359
private val requestContext: RequestContext,
5460
private val csvWriter: CsvWriter,
5561
) {
56-
@GetMapping("/aggregated")
62+
@GetMapping("/aggregated", produces = [MediaType.APPLICATION_JSON_VALUE])
5763
@LapisAggregatedResponse
5864
fun aggregated(
5965
@SequenceFilters
@@ -64,8 +70,7 @@ class LapisController(
6470
fields: List<String>?,
6571
@Parameter(
6672
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
67-
description = "The fields to order by." +
68-
" Fields specified here must either be \"count\" or also be present in \"fields\".",
73+
description = AGGREGATED_ORDER_BY_FIELDS_DESCRIPTION,
6974
)
7075
@RequestParam
7176
orderBy: List<OrderByField>?,
@@ -103,19 +108,161 @@ class LapisController(
103108

104109
requestContext.filter = request
105110

106-
return LapisResponse(siloQueryModel.aggregate(request))
111+
return LapisResponse(siloQueryModel.getAggregated(request))
107112
}
108113

109-
@PostMapping("/aggregated")
114+
@GetMapping("/aggregated", produces = [TEXT_CSV_HEADER])
110115
@LapisAggregatedResponse
116+
@Operation(
117+
description = AGGREGATED_ENDPONT_DESCRIPTION,
118+
operationId = "getAggregatedAsCsv",
119+
responses = [ApiResponse(responseCode = "200")],
120+
)
121+
fun getAggregatedAsCsv(
122+
@SequenceFilters
123+
@RequestParam
124+
sequenceFilters: Map<String, String>?,
125+
@Parameter(description = AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION)
126+
@RequestParam
127+
fields: List<String>?,
128+
@Parameter(
129+
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
130+
description = AGGREGATED_ORDER_BY_FIELDS_DESCRIPTION,
131+
)
132+
@RequestParam
133+
orderBy: List<OrderByField>?,
134+
@Parameter(
135+
schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_MUTATIONS_SCHEMA"),
136+
explode = Explode.TRUE,
137+
)
138+
@RequestParam
139+
nucleotideMutations: List<NucleotideMutation>?,
140+
@Parameter(schema = Schema(ref = "#/components/schemas/$AMINO_ACID_MUTATIONS_SCHEMA"))
141+
@RequestParam
142+
aminoAcidMutations: List<AminoAcidMutation>?,
143+
@Parameter(
144+
schema = Schema(ref = "#/components/schemas/$LIMIT_SCHEMA"),
145+
description = LIMIT_DESCRIPTION,
146+
)
147+
@RequestParam
148+
limit: Int? = null,
149+
@Parameter(
150+
schema = Schema(ref = "#/components/schemas/$OFFSET_SCHEMA"),
151+
description = OFFSET_DESCRIPTION,
152+
)
153+
@RequestParam
154+
offset: Int? = null,
155+
): String {
156+
val request = SequenceFiltersRequestWithFields(
157+
sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(),
158+
nucleotideMutations ?: emptyList(),
159+
aminoAcidMutations ?: emptyList(),
160+
fields ?: emptyList(),
161+
orderBy ?: emptyList(),
162+
limit,
163+
offset,
164+
)
165+
166+
return getResponseAsCsv(request, Delimiter.COMMA, siloQueryModel::getAggregated)
167+
}
168+
169+
@GetMapping("/aggregated", produces = [TEXT_TSV_HEADER])
170+
@LapisAggregatedResponse
171+
@Operation(
172+
description = AGGREGATED_ENDPONT_DESCRIPTION,
173+
operationId = "getAggregatedAsTsv",
174+
responses = [ApiResponse(responseCode = "200")],
175+
)
176+
fun getAggregatedAsTsv(
177+
@SequenceFilters
178+
@RequestParam
179+
sequenceFilters: Map<String, String>?,
180+
@Parameter(description = AGGREGATED_GROUP_BY_FIELDS_DESCRIPTION)
181+
@RequestParam
182+
fields: List<String>?,
183+
@Parameter(
184+
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
185+
description = AGGREGATED_ORDER_BY_FIELDS_DESCRIPTION,
186+
)
187+
@RequestParam
188+
orderBy: List<OrderByField>?,
189+
@Parameter(
190+
schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_MUTATIONS_SCHEMA"),
191+
explode = Explode.TRUE,
192+
)
193+
@RequestParam
194+
nucleotideMutations: List<NucleotideMutation>?,
195+
@Parameter(schema = Schema(ref = "#/components/schemas/$AMINO_ACID_MUTATIONS_SCHEMA"))
196+
@RequestParam
197+
aminoAcidMutations: List<AminoAcidMutation>?,
198+
@Parameter(
199+
schema = Schema(ref = "#/components/schemas/$LIMIT_SCHEMA"),
200+
description = LIMIT_DESCRIPTION,
201+
)
202+
@RequestParam
203+
limit: Int? = null,
204+
@Parameter(
205+
schema = Schema(ref = "#/components/schemas/$OFFSET_SCHEMA"),
206+
description = OFFSET_DESCRIPTION,
207+
)
208+
@RequestParam
209+
offset: Int? = null,
210+
): String {
211+
val request = SequenceFiltersRequestWithFields(
212+
sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(),
213+
nucleotideMutations ?: emptyList(),
214+
aminoAcidMutations ?: emptyList(),
215+
fields ?: emptyList(),
216+
orderBy ?: emptyList(),
217+
limit,
218+
offset,
219+
)
220+
221+
return getResponseAsCsv(request, Delimiter.TAB, siloQueryModel::getAggregated)
222+
}
223+
224+
@PostMapping("/aggregated", produces = [MediaType.APPLICATION_JSON_VALUE])
225+
@LapisAggregatedResponse
226+
@Operation(
227+
description = AGGREGATED_ENDPONT_DESCRIPTION,
228+
operationId = "postAggregated",
229+
)
111230
fun postAggregated(
112231
@Parameter(schema = Schema(ref = "#/components/schemas/$AGGREGATED_REQUEST_SCHEMA"))
113232
@RequestBody
114233
request: SequenceFiltersRequestWithFields,
115234
): LapisResponse<List<AggregationData>> {
116235
requestContext.filter = request
117236

118-
return LapisResponse(siloQueryModel.aggregate(request))
237+
return LapisResponse(siloQueryModel.getAggregated(request))
238+
}
239+
240+
@PostMapping("/aggregated", produces = [TEXT_CSV_HEADER])
241+
@Operation(
242+
description = AGGREGATED_ENDPONT_DESCRIPTION,
243+
operationId = "postAggregatedAsCsv",
244+
responses = [ApiResponse(responseCode = "200")],
245+
)
246+
fun postAggregatedAsCsv(
247+
@Parameter(schema = Schema(ref = "#/components/schemas/$DETAILS_REQUEST_SCHEMA"))
248+
@RequestBody
249+
request: SequenceFiltersRequestWithFields,
250+
): String {
251+
return getResponseAsCsv(request, Delimiter.COMMA, siloQueryModel::getAggregated)
252+
}
253+
254+
@PostMapping("/aggregated", produces = [TEXT_TSV_HEADER])
255+
@Operation(
256+
description = AGGREGATED_ENDPONT_DESCRIPTION,
257+
operationId = "postAggregatedAsTsv",
258+
responses = [ApiResponse(responseCode = "200")],
259+
)
260+
fun postAggregatedAsTsv(
261+
@Parameter(schema = Schema(ref = "#/components/schemas/$DETAILS_REQUEST_SCHEMA"))
262+
@RequestBody
263+
request: SequenceFiltersRequestWithFields,
264+
): String {
265+
return getResponseAsCsv(request, Delimiter.TAB, siloQueryModel::getAggregated)
119266
}
120267

121268
@GetMapping("/nucleotideMutations")
@@ -197,8 +344,7 @@ class LapisController(
197344
fields: List<String>?,
198345
@Parameter(
199346
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
200-
description = "The fields of the response to order by." +
201-
" Fields specified here must also be present in \"fields\".",
347+
description = DETAILS_ORDER_BY_FIELDS_DESCRIPTION,
202348
)
203349
@RequestParam
204350
orderBy: List<OrderByField>?,
@@ -251,8 +397,7 @@ class LapisController(
251397
fields: List<String>?,
252398
@Parameter(
253399
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
254-
description = "The fields of the response to order by." +
255-
" Fields specified here must also be present in \"fields\".",
400+
description = DETAILS_ORDER_BY_FIELDS_DESCRIPTION,
256401
)
257402
@RequestParam
258403
orderBy: List<OrderByField>?,
@@ -285,7 +430,7 @@ class LapisController(
285430
offset,
286431
)
287432

288-
return getDetailsAsCsv(request, Delimiter.COMMA)
433+
return getResponseAsCsv(request, Delimiter.COMMA, siloQueryModel::getDetails)
289434
}
290435

291436
@GetMapping("/details", produces = [TEXT_TSV_HEADER])
@@ -303,8 +448,7 @@ class LapisController(
303448
fields: List<String>?,
304449
@Parameter(
305450
schema = Schema(ref = "#/components/schemas/$ORDER_BY_FIELDS_SCHEMA"),
306-
description = "The fields of the response to order by." +
307-
" Fields specified here must also be present in \"fields\".",
451+
description = DETAILS_ORDER_BY_FIELDS_DESCRIPTION,
308452
)
309453
@RequestParam
310454
orderBy: List<OrderByField>?,
@@ -337,7 +481,7 @@ class LapisController(
337481
offset,
338482
)
339483

340-
return getDetailsAsCsv(request, Delimiter.TAB)
484+
return getResponseAsCsv(request, Delimiter.TAB, siloQueryModel::getDetails)
341485
}
342486

343487
@PostMapping("/details", produces = [MediaType.APPLICATION_JSON_VALUE])
@@ -367,7 +511,7 @@ class LapisController(
367511
@RequestBody
368512
request: SequenceFiltersRequestWithFields,
369513
): String {
370-
return getDetailsAsCsv(request, Delimiter.COMMA)
514+
return getResponseAsCsv(request, Delimiter.COMMA, siloQueryModel::getDetails)
371515
}
372516

373517
@PostMapping("/details", produces = [TEXT_TSV_HEADER])
@@ -381,20 +525,24 @@ class LapisController(
381525
@RequestBody
382526
request: SequenceFiltersRequestWithFields,
383527
): String {
384-
return getDetailsAsCsv(request, Delimiter.TAB)
528+
return getResponseAsCsv(request, Delimiter.TAB, siloQueryModel::getDetails)
385529
}
386530

387-
private fun getDetailsAsCsv(request: SequenceFiltersRequestWithFields, delimiter: Delimiter): String {
531+
private fun getResponseAsCsv(
532+
request: SequenceFiltersRequestWithFields,
533+
delimiter: Delimiter,
534+
getResponse: (request: SequenceFiltersRequestWithFields) -> List<CsvRecord>,
535+
): String {
388536
requestContext.filter = request
389537

390-
val data = siloQueryModel.getDetails(request)
538+
val data = getResponse(request)
391539

392540
if (data.isEmpty()) {
393541
return ""
394542
}
395543

396-
val headers = data[0].keys.toTypedArray<String>()
397-
return csvWriter.write(headers, data.map { it.asCsvRecord() }, delimiter)
544+
val headers = data[0].asHeader()
545+
return csvWriter.write(headers, data, delimiter)
398546
}
399547
}
400548

‎lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.genspectrum.lapis.model
22

33
import org.genspectrum.lapis.request.MutationProportionsRequest
44
import org.genspectrum.lapis.request.SequenceFiltersRequestWithFields
5+
import org.genspectrum.lapis.response.DetailsData
56
import org.genspectrum.lapis.silo.SiloAction
67
import org.genspectrum.lapis.silo.SiloClient
78
import org.genspectrum.lapis.silo.SiloQuery
@@ -13,7 +14,7 @@ class SiloQueryModel(
1314
private val siloFilterExpressionMapper: SiloFilterExpressionMapper,
1415
) {
1516

16-
fun aggregate(sequenceFilters: SequenceFiltersRequestWithFields) = siloClient.sendQuery(
17+
fun getAggregated(sequenceFilters: SequenceFiltersRequestWithFields) = siloClient.sendQuery(
1718
SiloQuery(
1819
SiloAction.aggregated(
1920
sequenceFilters.fields,
@@ -38,7 +39,7 @@ class SiloQueryModel(
3839
),
3940
)
4041

41-
fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields) = siloClient.sendQuery(
42+
fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields): List<DetailsData> = siloClient.sendQuery(
4243
SiloQuery(
4344
SiloAction.details(
4445
sequenceFilters.fields,

‎lapis2/src/main/kotlin/org/genspectrum/lapis/response/SiloResponse.kt

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,38 @@ import com.fasterxml.jackson.databind.JsonNode
88
import com.fasterxml.jackson.databind.JsonSerializer
99
import com.fasterxml.jackson.databind.SerializerProvider
1010
import io.swagger.v3.oas.annotations.media.Schema
11+
import org.genspectrum.lapis.controller.CsvRecord
1112
import org.springframework.boot.jackson.JsonComponent
1213

1314
const val COUNT_PROPERTY = "count"
1415

15-
data class AggregationData(val count: Int, @Schema(hidden = true) val fields: Map<String, JsonNode>)
16+
data class AggregationData(val count: Int, @Schema(hidden = true) val fields: Map<String, JsonNode>) : CsvRecord {
17+
override fun asArray() = fields.values.map { it.asText() }.plus(count.toString()).toTypedArray()
18+
override fun asHeader() = fields.keys.plus(COUNT_PROPERTY).toTypedArray()
19+
}
20+
21+
data class DetailsData(val map: Map<String, JsonNode>) : Map<String, JsonNode> by map, CsvRecord {
22+
override fun asArray() = values.map { it.asText() }.toTypedArray()
23+
override fun asHeader() = keys.toTypedArray()
24+
}
25+
26+
@JsonComponent
27+
class DetailsDataSerializer : JsonSerializer<DetailsData>() {
28+
override fun serialize(value: DetailsData, gen: JsonGenerator, serializers: SerializerProvider) {
29+
gen.writeStartObject()
30+
value.forEach { (key, value) -> gen.writeObjectField(key, value) }
31+
gen.writeEndObject()
32+
}
33+
}
34+
35+
@JsonComponent
36+
class DetailsDataDeserializer : JsonDeserializer<DetailsData>() {
37+
override fun deserialize(p: JsonParser, ctxt: DeserializationContext): DetailsData {
38+
val node = p.readValueAsTree<JsonNode>()
39+
val fields = node.fields().asSequence().associate { it.key to it.value }
40+
return DetailsData(fields)
41+
}
42+
}
1643

1744
@JsonComponent
1845
class AggregationDataSerializer : JsonSerializer<AggregationData>() {

0 commit comments

Comments
 (0)