Skip to content
This repository was archived by the owner on Oct 5, 2024. It is now read-only.

Commit b581efe

Browse files
emi420emi420
authored
Merge pull request #501 from hotosm/dev
Handle multiple databases, improvements for data management
2 parents ad61a24 + 4f7f9e6 commit b581efe

File tree

14 files changed

+336
-196
lines changed

14 files changed

+336
-196
lines changed

‎setup/db/indexes.sql

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,6 @@ CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC);
22
CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC);
33
CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC);
44
CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC);
5-
CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id);
6-
CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id);
7-
CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id);
8-
CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id);
95

106
CREATE INDEX nodes_version_idx ON public.nodes (version);
117
CREATE INDEX ways_poly_version_idx ON public.ways_poly (version);

‎setup/db/underpass.sql

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -109,25 +109,10 @@ CREATE TABLE IF NOT EXISTS public.relations (
109109
ALTER TABLE ONLY public.relations
110110
ADD CONSTRAINT relations_pkey PRIMARY KEY (osm_id);
111111

112-
CREATE TABLE IF NOT EXISTS public.way_refs (
113-
way_id int8,
114-
node_id int8
115-
);
116-
117-
CREATE TABLE IF NOT EXISTS public.rel_refs (
118-
rel_id int8,
119-
way_id int8
120-
);
121-
122112
CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC);
123113
CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC);
124114
CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC);
125115
CREATE UNIQUE INDEX relations_id_idx ON public.relations(osm_id DESC);
126-
CREATE INDEX way_refs_node_id_idx ON public.way_refs (node_id);
127-
CREATE INDEX way_refs_way_id_idx ON public.way_refs (way_id);
128-
129-
CREATE INDEX rel_refs_rel_id_idx ON public.rel_refs (rel_id);
130-
CREATE INDEX rel_refs_way_id_idx ON public.rel_refs (way_id);
131116

132117
CREATE INDEX nodes_version_idx ON public.nodes (version);
133118
CREATE INDEX ways_poly_version_idx ON public.ways_poly (version);

‎src/bootstrap/bootstrap.cc

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,12 @@ BootstrapQueries
5353
Bootstrap::allTasksQueries(std::shared_ptr<std::vector<BootstrapTask>> tasks) {
5454
BootstrapQueries queries;
5555
for (auto it = tasks->begin(); it != tasks->end(); ++it) {
56-
queries.underpass += it->query ;
57-
queries.osm += it->osmquery ;
56+
for (auto itt = it->query.begin(); itt != it->query.end(); ++itt) {
57+
queries.underpass.push_back(*itt);
58+
}
59+
for (auto itt = it->osmquery.begin(); itt != it->osmquery.end(); ++itt) {
60+
queries.osm.push_back(*itt);
61+
}
5862
}
5963
return queries;
6064
}
@@ -154,8 +158,12 @@ Bootstrap::processWays() {
154158

155159
auto queries = allTasksQueries(tasks);
156160

157-
db->query(queries.underpass);
158-
osmdb->query(queries.osm);
161+
for (auto it = queries.underpass.begin(); it != queries.underpass.end(); ++it) {
162+
db->query(*it);
163+
}
164+
for (auto it = queries.osm.begin(); it != queries.osm.end(); ++it) {
165+
osmdb->query(*it);
166+
}
159167

160168
lastid = ways->back().id;
161169
for (auto it = tasks->begin(); it != tasks->end(); ++it) {
@@ -206,8 +214,12 @@ Bootstrap::processNodes() {
206214
pool.join();
207215

208216
auto queries = allTasksQueries(tasks);
209-
db->query(queries.underpass);
210-
osmdb->query(queries.osm);
217+
for (auto it = queries.underpass.begin(); it != queries.underpass.end(); ++it) {
218+
db->query(*it);
219+
}
220+
for (auto it = queries.osm.begin(); it != queries.osm.end(); ++it) {
221+
osmdb->query(*it);
222+
}
211223
lastid = nodes->back().id;
212224
for (auto it = tasks->begin(); it != tasks->end(); ++it) {
213225
count += it->processed;
@@ -256,9 +268,12 @@ Bootstrap::processRelations() {
256268
pool.join();
257269

258270
auto queries = allTasksQueries(tasks);
259-
db->query(queries.underpass);
260-
osmdb->query(queries.osm);
261-
271+
for (auto it = queries.underpass.begin(); it != queries.underpass.end(); ++it) {
272+
db->query(*it);
273+
}
274+
for (auto it = queries.osm.begin(); it != queries.osm.end(); ++it) {
275+
osmdb->query(*it);
276+
}
262277
lastid = relations->back().id;
263278
for (auto it = tasks->begin(); it != tasks->end(); ++it) {
264279
count += it->processed;
@@ -292,16 +307,15 @@ Bootstrap::threadBootstrapWayTask(WayTask wayTask)
292307
if (i < ways->size()) {
293308
auto way = ways->at(i);
294309
wayval->push_back(validator->checkWay(way, "building"));
295-
// Fill the way_refs table
296-
if (!norefs) {
297-
for (auto ref = way.refs.begin(); ref != way.refs.end(); ++ref) {
298-
task.osmquery += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way.id) + "," + std::to_string(*ref) + "); ";
299-
}
300-
}
301310
++processed;
302311
}
303312
}
304-
queryvalidate->ways(wayval, task.query);
313+
314+
auto result = queryvalidate->ways(wayval);
315+
for (auto it = result->begin(); it != result->end(); ++it) {
316+
task.query.push_back(*it);
317+
}
318+
305319
task.processed = processed;
306320
const std::lock_guard<std::mutex> lock(tasks_change_mutex);
307321
(*tasks)[taskIndex] = task;
@@ -337,7 +351,12 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask)
337351
++processed;
338352
}
339353
}
340-
queryvalidate->nodes(nodeval, task.query);
354+
355+
auto result = queryvalidate->nodes(nodeval);
356+
for (auto it = result->begin(); it != result->end(); ++it) {
357+
task.query.push_back(*it);
358+
}
359+
341360
task.processed = processed;
342361
const std::lock_guard<std::mutex> lock(tasks_change_mutex);
343362
(*tasks)[taskIndex] = task;
@@ -367,7 +386,7 @@ Bootstrap::threadBootstrapRelationTask(RelationTask relationTask)
367386
// relationval->push_back(validator->checkRelation(way, "building"));
368387
// Fill the rel_refs table
369388
for (auto mit = relation.members.begin(); mit != relation.members.end(); ++mit) {
370-
task.osmquery += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); ";
389+
task.osmquery.push_back("INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(relation.id) + "," + std::to_string(mit->ref) + "); ");
371390
}
372391
++processed;
373392
}

‎src/bootstrap/bootstrap.hh

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,18 @@ namespace bootstrap {
3333
/// \struct BootstrapTask
3434
/// \brief Represents a bootstrap task
3535
struct BootstrapTask {
36-
std::string query = "";
37-
std::string osmquery = "";
36+
std::vector<std::string> query;
37+
std::vector<std::string> osmquery;
3838
int processed = 0;
3939
};
4040

4141
/// \struct BootstrapQueries
4242
/// \brief Represents a bootstrap queries list
4343
struct BootstrapQueries {
44-
std::string underpass = "";
45-
std::string osm = "";
44+
std::vector<std::string> underpass;
45+
std::vector<std::string> osm;
4646
};
4747

48-
4948
struct WayTask {
5049
int taskIndex;
5150
std::shared_ptr<std::vector<BootstrapTask>> tasks;

‎src/data/pq.cc

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,15 @@ Pq::query(const std::string &query)
152152
{
153153
std::scoped_lock write_lock{pqxx_mutex};
154154
pqxx::work worker(*sdb);
155-
auto result = worker.exec(query);
156-
worker.commit();
155+
pqxx::result result;
156+
try {
157+
result = worker.exec(query);
158+
worker.commit();
159+
} catch (std::exception &e) {
160+
log_error("ERROR executing query %1%", e.what());
161+
// Return an empty result so higher level code can handle the error
162+
return result;
163+
}
157164
return result;
158165
}
159166

‎src/osm/osmchange.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -697,7 +697,7 @@ OsmChange::dump(void)
697697
// rel->dump();
698698
// }
699699
// }
700-
std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl;
700+
// std::cerr << "Final timestamp: " << to_simple_string(final_entry) << std::endl;
701701

702702
}
703703

0 commit comments

Comments
 (0)