Skip to content

Commit 21c0433

Browse files
red-fenixred-fenix
authored
Add option for polygon/linestring in results (#823)
At the moment, photon only returns the point of a location, and not the polygon (see #259). This PR will add the option to add the polygon (i.e. geometry) to the Elasticsearch Index and a API parameter polygon to return said polygon. If no polygon exists, the point is returned. WARNING: This will increase the Elasticsearch Index size! (~575GB for a Planet import). To enable: add the command line argument -use-geometry-column whilst importing and add &geometry=true to the API call.
1 parent bf7798c commit 21c0433

File tree

58 files changed

+632
-130
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+632
-130
lines changed

‎app/es_embedded/es/mappings.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949
"coordinate": {
5050
"type": "geo_point"
5151
},
52+
"geometry": {
53+
"type": "geo_shape"
54+
},
5255
"country": {
5356
"properties": {
5457
"default": {

‎app/es_embedded/src/main/java/de/komoot/photon/Server.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ public class Server {
5656
private static final String FIELD_VERSION = "database_version";
5757
private static final String FIELD_LANGUAGES = "indexed_languages";
5858
private static final String FIELD_IMPORT_DATE = "import_date";
59+
private static final String FIELD_SUPPORT_GEOMETRIES = "support_geometries";
5960

6061
private Node esNode;
6162

@@ -177,14 +178,18 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx
177178

178179
}
179180

180-
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
181+
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
181182
deleteIndex();
182183

183184
loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);
184185

185186
createAndPutIndexMapping(languages, supportStructuredQueries);
186187

187-
DatabaseProperties dbProperties = new DatabaseProperties(languages, importDate, false);
188+
DatabaseProperties dbProperties = new DatabaseProperties()
189+
.setLanguages(languages)
190+
.setImportDate(importDate)
191+
.setSupportGeometries(supportGeometries);
192+
188193
saveToDatabase(dbProperties);
189194

190195
return dbProperties;
@@ -239,6 +244,7 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException
239244
.field(FIELD_VERSION, DATABASE_VERSION)
240245
.field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages()))
241246
.field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null)
247+
.field(FIELD_SUPPORT_GEOMETRIES, Boolean.toString(dbProperties.getSupportGeometries()))
242248
.endObject().endObject();
243249

244250
esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
@@ -276,11 +282,15 @@ public DatabaseProperties loadFromDatabase() {
276282
}
277283

278284
String langString = properties.get(FIELD_LANGUAGES);
285+
279286
String importDateString = properties.get(FIELD_IMPORT_DATE);
280287

288+
String supportGeometries = properties.get(FIELD_SUPPORT_GEOMETRIES);
289+
281290
return new DatabaseProperties(langString == null ? null : langString.split(","),
282-
importDateString == null ? null : Date.from(Instant.parse(importDateString)),
283-
false);
291+
importDateString == null ? null : Date.from(Instant.parse(importDateString)),
292+
false,
293+
Boolean.parseBoolean(supportGeometries));
284294
}
285295

286296
public Importer createImporter(String[] languages, String[] extraTags) {

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/ElasticResult.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@ public double[] getCoordinates() {
6666
return new double[]{coordinate.get(Constants.LON), coordinate.get(Constants.LAT)};
6767
}
6868

69+
@Override
70+
public String getGeometry() {
71+
return (String) result.getSource().get("geometry");
72+
}
73+
6974
@Override
7075
public double[] getExtent() {
7176
final Map<String, Object> extent = (Map<String, Object>) result.getSource().get("extent");

‎app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33
import de.komoot.photon.Constants;
44
import de.komoot.photon.PhotonDoc;
55
import de.komoot.photon.nominatim.model.AddressType;
6-
import org.elasticsearch.common.xcontent.XContentBuilder;
7-
import org.elasticsearch.common.xcontent.XContentFactory;
6+
7+
import org.elasticsearch.common.xcontent.*;
8+
import org.elasticsearch.common.xcontent.json.JsonXContent;
89
import org.locationtech.jts.geom.Envelope;
10+
import org.locationtech.jts.io.geojson.GeoJsonWriter;
911

1012
import java.io.IOException;
1113
import java.util.HashMap;
@@ -38,6 +40,17 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, String[
3840
.endObject();
3941
}
4042

43+
if (doc.getGeometry() != null) {
44+
GeoJsonWriter g = new GeoJsonWriter();
45+
46+
XContentParser parser = JsonXContent
47+
.jsonXContent
48+
.createParser(NamedXContentRegistry.EMPTY, g.write(doc.getGeometry()));
49+
50+
builder.field("geometry");
51+
builder.copyCurrentStructure(parser);
52+
}
53+
4154
if (doc.getHouseNumber() != null) {
4255
builder.field("housenumber", doc.getHouseNumber());
4356
}

‎app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
import de.komoot.photon.searcher.PhotonResult;
88
import org.junit.jupiter.api.AfterEach;
99
import org.junit.jupiter.api.io.TempDir;
10+
import org.locationtech.jts.io.ParseException;
11+
import org.locationtech.jts.io.WKTReader;
1012

1113
import java.io.IOException;
1214
import java.nio.file.Path;
@@ -25,9 +27,9 @@ public class ESBaseTester {
2527

2628
private ElasticTestServer server;
2729

28-
protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) {
30+
protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) throws ParseException {
2931
Point location = FACTORY.createPoint(new Coordinate(lon, lat));
30-
return new PhotonDoc(id, "W", osmId, key, value).names(Collections.singletonMap("name", "berlin")).centroid(location);
32+
return new PhotonDoc(id, "W", osmId, key, value).names(Collections.singletonMap("name", "berlin")).centroid(location).geometry(new WKTReader().read("POLYGON ((6.4440619 52.1969454, 6.4441094 52.1969158, 6.4441408 52.1969347, 6.4441138 52.1969516, 6.4440933 52.1969643, 6.4440619 52.1969454))"));
3133
}
3234

3335
protected PhotonResult getById(int id) {
@@ -45,17 +47,21 @@ public void tearDown() throws IOException {
4547
}
4648

4749
public void setUpES() throws IOException {
48-
setUpES(dataDirectory, "en");
50+
setUpES(dataDirectory, false,"en");
51+
}
52+
53+
public void setUpESWithGeometry() throws IOException {
54+
setUpES(dataDirectory, true,"en");
4955
}
5056
/**
5157
* Setup the ES server
5258
*
5359
* @throws IOException
5460
*/
55-
public void setUpES(Path testDirectory, String... languages) throws IOException {
61+
public void setUpES(Path testDirectory, boolean supportGeometries, String... languages) throws IOException {
5662
server = new ElasticTestServer(testDirectory.toString());
5763
server.start(TEST_CLUSTER_NAME, new String[]{});
58-
server.recreateIndex(languages, new Date(), false);
64+
server.recreateIndex(languages, new Date(), false, supportGeometries);
5965
refresh();
6066
}
6167

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticGetIdResult.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ public double[] getCoordinates() {
3434
throw new NotImplementedException();
3535
}
3636

37+
public String getGeometry() {
38+
throw new NotImplementedException();
39+
}
40+
3741
@Override
3842
public double[] getExtent() {
3943
throw new NotImplementedException();

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String
4444

4545
@BeforeAll
4646
void setUp() throws Exception {
47-
setUpES(instanceTestDirectory, "en", "de", "fr", "it");
47+
setUpES(instanceTestDirectory, false, "en", "de", "fr", "it");
4848
Importer instance = getServer().createImporter(new String[]{"en", "de", "fr", "it"},
4949
new String[]{"population", "capital"});
5050

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ImporterTest.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import de.komoot.photon.searcher.PhotonResult;
77
import org.junit.jupiter.api.BeforeEach;
88
import org.junit.jupiter.api.Test;
9+
import org.locationtech.jts.io.ParseException;
10+
import org.locationtech.jts.io.WKTReader;
911

1012
import java.io.IOException;
1113
import java.util.Collections;
@@ -22,10 +24,11 @@ public void setUp() throws IOException {
2224
}
2325

2426
@Test
25-
void testAddSimpleDoc() {
27+
void testAddSimpleDoc() throws ParseException {
2628
Importer instance = makeImporterWithExtra("");
2729

2830
instance.add(new PhotonDoc(1234, "N", 1000, "place", "city")
31+
.geometry(new WKTReader().read("MULTIPOLYGON (((6.111933 51.2659309, 6.1119417 51.2659247, 6.1119554 51.2659249, 6.1119868 51.2659432, 6.111964 51.2659591, 6.1119333 51.2659391, 6.111933 51.2659309)))"))
2932
.extraTags(Collections.singletonMap("maxspeed", "100")), 0);
3033
instance.finish();
3134
refresh();

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ void testSaveAndLoadFromDatabase() throws IOException {
1616
setUpES();
1717

1818
Date now = new Date();
19-
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false);
19+
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false, false);
2020
getServer().saveToDatabase(prop);
2121

2222
prop = getServer().loadFromDatabase();

‎app/opensearch/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ dependencies {
2222
implementation 'org.apache.httpcomponents.client5:httpclient5:5.4.1'
2323
implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2'
2424

25-
implementation('org.codelibs.opensearch:opensearch-runner:2.18.0.0') {
25+
implementation('org.codelibs.opensearch:opensearch-runner:2.18.0.1') {
2626
exclude(module: 'repository-url')
2727
exclude(module: 'reindex-client')
2828
exclude(module: 'rank-eval-client')

‎app/opensearch/src/main/java/de/komoot/photon/Server.java

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,11 @@ public class Server {
3333

3434
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);
3535

36-
public static final String OPENSEARCH_MODULES =
37-
"org.opensearch.transport.Netty4Plugin,"
38-
+ "org.opensearch.analysis.common.CommonAnalysisPlugin";
36+
// public static final String OPENSEARCH_MODULES =
37+
// "org.opensearch.transport.Netty4Plugin,"
38+
// + "org.opensearch.analysis.common.CommonAnalysisPlugin,"
39+
// + "org.opensearch.geo.GeoModulePlugin,"
40+
// + "org.opensearch.geospatial.plugin.GeospatialPlugin";
3941

4042
protected OpenSearchClient client;
4143
private OpenSearchRunner runner = null;
@@ -86,7 +88,6 @@ private HttpHost[] startInternal(String clusterName) {
8688
.basePath(dataDirectory)
8789
.clusterName(clusterName)
8890
.numOfNode(1)
89-
.moduleTypes(OPENSEARCH_MODULES)
9091
);
9192

9293
runner.ensureYellow();
@@ -119,7 +120,7 @@ public void shutdown() {
119120
}
120121
}
121122

122-
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
123+
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
123124
// delete any existing data
124125
if (client.indices().exists(e -> e.index(PhotonIndex.NAME)).value()) {
125126
client.indices().delete(d -> d.index(PhotonIndex.NAME));
@@ -129,7 +130,7 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo
129130

130131
(new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
131132

132-
var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries);
133+
var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries, supportGeometries);
133134
saveToDatabase(dbProperties);
134135

135136
return dbProperties;
@@ -180,7 +181,8 @@ public DatabaseProperties loadFromDatabase() throws IOException {
180181

181182
return new DatabaseProperties(dbEntry.source().languages,
182183
dbEntry.source().importDate,
183-
dbEntry.source().supportStructuredQueries);
184+
dbEntry.source().supportStructuredQueries,
185+
dbEntry.source().supportGeometries);
184186
}
185187

186188
public Importer createImporter(String[] languages, String[] extraTags) {

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ public class DBPropertyEntry {
99
public Date importDate;
1010
public String[] languages;
1111
public boolean supportStructuredQueries;
12+
public boolean supportGeometries;
1213

1314
public DBPropertyEntry() {}
1415

@@ -17,5 +18,6 @@ public DBPropertyEntry(DatabaseProperties props, String databaseVersion) {
1718
importDate = props.getImportDate();
1819
languages = props.getLanguages();
1920
supportStructuredQueries = props.getSupportStructuredQueries();
21+
supportGeometries = props.getSupportGeometries();
2022
}
2123
}

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/Importer.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import org.opensearch.client.opensearch.OpenSearchClient;
55
import org.opensearch.client.opensearch._types.Time;
66
import org.opensearch.client.opensearch.core.BulkRequest;
7+
import org.opensearch.client.opensearch.core.bulk.BulkOperation;
8+
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
79
import org.slf4j.Logger;
810

911
import java.io.IOException;
@@ -54,7 +56,14 @@ private void saveDocuments() {
5456
var response = client.bulk(bulkRequest.build());
5557

5658
if (response.errors()) {
57-
LOGGER.error("Error during bulk import.");
59+
for (BulkResponseItem bri: response.items()) {
60+
LOGGER.error("Error during bulk import.");
61+
if (bri.error() != null) {
62+
LOGGER.error(bri.error().reason());
63+
LOGGER.error(bri.error().type());
64+
LOGGER.error(bri.error().stackTrace());
65+
}
66+
}
5867
}
5968
} catch (IOException e) {
6069
LOGGER.error("Error during bulk import", e);

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/IndexMapping.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ private void setupBaseMappings() {
7878
}
7979

8080
mappings.properties("coordinate", b -> b.geoPoint(p -> p));
81+
mappings.properties("geometry", b -> b.geoShape(p -> p));
8182
mappings.properties("countrycode", b -> b.keyword(p -> p.index(true)));
8283
mappings.properties("importance", b -> b.float_(p -> p.index(false)));
8384

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,16 @@ public class OpenSearchResult implements PhotonResult {
1111
private double score = 0.0;
1212
private final double[] extent;
1313
private final double[] coordinates;
14+
private final String geometry;
1415
private final Map<String, Object> infos;
1516
private final Map<String, Map<String, String>> localeTags;
1617

17-
OpenSearchResult(double[] extent, double[] coordinates, Map<String, Object> infos, Map<String, Map<String, String>> localeTags) {
18+
OpenSearchResult(double[] extent, double[] coordinates, Map<String, Object> infos, Map<String, Map<String, String>> localeTags, String geometry) {
1819
this.extent = extent;
1920
this.coordinates = coordinates;
2021
this.infos = infos;
2122
this.localeTags = localeTags;
23+
this.geometry = geometry;
2224
}
2325

2426
public OpenSearchResult setScore(double score) {
@@ -61,6 +63,10 @@ public double[] getCoordinates() {
6163
return coordinates;
6264
}
6365

66+
public String getGeometry() {
67+
return geometry;
68+
}
69+
6470
@Override
6571
public double[] getExtent() {
6672
return extent;

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResultDeserializer.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) t
2929
final Map<String, Object> tags = new HashMap<>();
3030
final Map<String, Map<String, String>> localeTags = new HashMap<>();
3131

32+
String geometry = null;
33+
if (node.get("geometry") != null) {
34+
geometry = node.get("geometry").toString();
35+
}
36+
3237
var fields = node.fields();
3338
while (fields.hasNext()) {
3439
final var entry = fields.next();
@@ -55,7 +60,7 @@ public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) t
5560
}
5661
}
5762

58-
return new OpenSearchResult(extent, coordinates, tags, localeTags);
63+
return new OpenSearchResult(extent, coordinates, tags, localeTags, geometry);
5964
}
6065

6166
private double[] extractExtent(ObjectNode node) {
@@ -78,5 +83,4 @@ private double[] extractCoordinate(ObjectNode node) {
7883

7984
return new double[]{node.get(Constants.LON).doubleValue(), node.get(Constants.LAT).doubleValue()};
8085
}
81-
8286
}

0 commit comments

Comments
 (0)