feat(lapis): expose the lineage definition files used by SILO

resolves #1034

Author: fengelniederhammer

README.md

@@ -51,6 +51,7 @@ Higher versions will also work if they are not specified in the table.
 
 | LAPIS  | SILO   |
 |--------|--------|
+| 0.3.14 | 0.5.3  |
 | 0.3.13 | 0.5.2  |
 | 0.3.7  | 0.3.0  |
 | 0.2.10 | 0.2.14 |

lapis-e2e/test/info.spec.ts

@@ -1,12 +1,10 @@
 import { expect } from 'chai';
 import { lapisInfoClient } from './common';
 
-describe('The info endpoind', () => {
+describe('The info endpoint', () => {
   it('should return all infos', async () => {
     const info = await lapisInfoClient.getInfo();
 
-    console.log(info);
-
     expect(info.dataVersion).to.match(/\d+/);
     expect(info.lapisVersion).to.be.not.empty;
     expect(info.siloVersion).to.be.not.empty;

lapis-e2e/test/lineageDefinition.spec.ts

@@ -0,0 +1,41 @@
+import { expect } from 'chai';
+import { basePath, lapisInfoClient } from './common';
+
+describe('The lineageDefinition endpoint', () => {
+  it('should return the file as JSON', async () => {
+    const lineageDefinition = await lapisInfoClient.getLineageDefinition({
+      column: 'pango_lineage',
+    });
+
+    expect(lineageDefinition['A']).to.deep.equal({
+      parents: undefined,
+      aliases: undefined,
+    });
+    expect(lineageDefinition['A.1']).to.deep.equal({
+      parents: ['A'],
+      aliases: undefined,
+    });
+    expect(lineageDefinition['AT.1']).to.deep.equal({
+      parents: ['B.1.1.370'],
+      aliases: ['B.1.1.370.1'],
+    });
+  });
+
+  it('should return the file as YAML', async () => {
+    const result = await fetch(basePath + '/sample/lineageDefinition/pango_lineage', {
+      headers: { Accept: 'application/yaml' },
+    });
+    const lineageDefinitionYaml = await result.text();
+
+    const expectedFileStart = `---
+A: {}
+A.1:
+  parents:
+  - "A"
+A.11:
+  parents:
+  - "A"`;
+
+    expect(lineageDefinitionYaml).to.match(new RegExp(`^${expectedFileStart}`));
+  });
+});

lapis/src/main/kotlin/org/genspectrum/lapis/controller/ControllerDescriptions.kt

@@ -21,6 +21,9 @@ const val AMINO_ACID_INSERTIONS_ENDPOINT_DESCRIPTION =
     considered."""
 const val INFO_ENDPOINT_DESCRIPTION = "Returns information about LAPIS."
 const val DATABASE_CONFIG_ENDPOINT_DESCRIPTION = "Returns the database configuration."
+const val LINEAGE_DEFINITION_ENDPOINT_DESCRIPTION = """Download the lineage definition file used for a certain column.
+This can be used to reconstruct the lineage tree. 
+"""
 const val REFERENCE_GENOME_ENDPOINT_DESCRIPTION = "Returns the reference genome."
 const val ALIGNED_AMINO_ACID_SEQUENCE_ENDPOINT_DESCRIPTION =
     """Returns a string of aligned amino acid sequences. Only sequences matching the specified

lapis/src/main/kotlin/org/genspectrum/lapis/controller/InfoController.kt

@@ -9,13 +9,16 @@ import org.genspectrum.lapis.logging.RequestIdContext
 import org.genspectrum.lapis.model.SiloQueryModel
 import org.genspectrum.lapis.response.LapisInfo
 import org.genspectrum.lapis.response.LapisInfoFactory
+import org.genspectrum.lapis.silo.LineageDefinition
 import org.springframework.http.MediaType
 import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RestController
 
 const val INFO_ROUTE = "/info"
 const val DATABASE_CONFIG_ROUTE = "/databaseConfig"
+const val LINEAGE_DEFINITION_ROUTE = "/lineageDefinition"
 const val REFERENCE_GENOME_ROUTE = "/referenceGenome"
 
 @RestController
@@ -45,6 +48,15 @@ class InfoController(
     @Operation(description = DATABASE_CONFIG_ENDPOINT_DESCRIPTION)
     fun getDatabaseConfigAsJson(): DatabaseConfig = databaseConfig
 
+    @GetMapping(
+        "$LINEAGE_DEFINITION_ROUTE/{column}",
+        produces = [MediaType.APPLICATION_JSON_VALUE, APPLICATION_YAML_VALUE],
+    )
+    @Operation(description = LINEAGE_DEFINITION_ENDPOINT_DESCRIPTION)
+    fun getLineageDefinition(
+        @PathVariable("column") column: String,
+    ): LineageDefinition = siloQueryModel.getLineageDefinition(column)
+
     @GetMapping(REFERENCE_GENOME_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE])
     @Operation(description = REFERENCE_GENOME_ENDPOINT_DESCRIPTION)
     fun getReferenceGenome(): ReferenceGenome = referenceGenome

lapis/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt

@@ -178,4 +178,6 @@ class SiloQueryModel(
     )
 
     fun getInfo(): InfoData = siloClient.callInfo()
+
+    fun getLineageDefinition(column: String) = siloClient.getLineageDefinition(column)
 }

lapis/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt

@@ -1,12 +1,14 @@
 package org.genspectrum.lapis.silo
 
+import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import mu.KotlinLogging
 import org.genspectrum.lapis.controller.LapisHeaders.REQUEST_ID
 import org.genspectrum.lapis.logging.RequestContext
 import org.genspectrum.lapis.logging.RequestIdContext
 import org.genspectrum.lapis.response.InfoData
+import org.genspectrum.lapis.util.YamlObjectMapper
 import org.springframework.cache.annotation.Cacheable
 import org.springframework.http.HttpHeaders
 import org.springframework.http.HttpStatus
@@ -52,6 +54,12 @@ class SiloClient(
         dataVersion.dataVersion = info.dataVersion
         return info
     }
+
+    fun getLineageDefinition(column: String): LineageDefinition {
+        log.info { "Calling SILO lineageDefinition for column '$column'" }
+
+        return cachedSiloClient.getLineageDefinition(column)
+    }
 }
 
 const val SILO_QUERY_CACHE_NAME = "siloQueryCache"
@@ -60,6 +68,7 @@ const val SILO_QUERY_CACHE_NAME = "siloQueryCache"
 open class CachedSiloClient(
     private val siloUris: SiloUris,
     private val objectMapper: ObjectMapper,
+    private val yamlObjectMapper: YamlObjectMapper,
     private val requestIdContext: RequestIdContext,
     private val requestContext: RequestContext,
 ) {
@@ -114,6 +123,16 @@ open class CachedSiloClient(
         )
     }
 
+    fun getLineageDefinition(column: String): LineageDefinition {
+        val response = send(
+            uri = siloUris.lineageDefinition(column),
+            bodyHandler = BodyHandlers.ofString(),
+            tryToReadSiloErrorFromBody = ::tryToReadSiloErrorFromString,
+        ) { it.GET() }
+
+        return yamlObjectMapper.objectMapper.readValue(response.body())
+    }
+
     private fun <ResponseBodyType> send(
         uri: URI,
         bodyHandler: HttpResponse.BodyHandler<ResponseBodyType>,
@@ -193,3 +212,11 @@ data class SiloErrorResponse(val error: String, val message: String)
 data class SiloInfo(
     val version: String,
 )
+
+typealias LineageDefinition = Map<String, LineageNode>
+
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
+data class LineageNode(
+    val parents: List<String>?,
+    val aliases: List<String>?,
+)

lapis/src/main/kotlin/org/genspectrum/lapis/silo/SiloUris.kt

@@ -6,8 +6,10 @@ import java.net.URI
 
 @Component
 class SiloUris(
-    @Value("\${silo.url}") siloUrl: String,
+    @Value("\${silo.url}") private val siloUrl: String,
 ) {
     val query = URI("$siloUrl/query")
     val info = URI("$siloUrl/info")
+
+    fun lineageDefinition(column: String): URI = URI("$siloUrl/lineageDefinition/").resolve(column)
 }

lapis/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt

@@ -435,6 +435,48 @@ class SiloClientTest(
         assertThat(exception.message, containsString(errorMessage))
     }
 
+    @Test
+    fun `get lineage definition`() {
+        val columnName = "test_column"
+        MockServerClient("localhost", MOCK_SERVER_PORT)
+            .`when`(
+                request()
+                    .withMethod("GET")
+                    .withPath("/lineageDefinition/$columnName")
+                    .withHeader("X-Request-Id", REQUEST_ID_VALUE),
+            )
+            .respond(
+                response()
+                    .withStatusCode(200)
+                    .withBody(
+                        """
+                            A: {}
+                            A.1:
+                              parents:
+                              - A
+                            B:
+                              aliases:
+                              - A.1.1
+                              parents:
+                              - A.1
+                        """.trimIndent(),
+                    ),
+            )
+
+        val actual = underTest.getLineageDefinition(columnName)
+
+        assertThat(
+            actual,
+            equalTo(
+                mapOf(
+                    "A" to LineageNode(parents = null, aliases = null),
+                    "A.1" to LineageNode(parents = listOf("A"), aliases = null),
+                    "B" to LineageNode(parents = listOf("A.1"), aliases = listOf("A.1.1")),
+                ),
+            ),
+        )
+    }
+
     companion object {
         @JvmStatic
         val mutationActions = listOf(