First commit

Author: akidee

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+snipplets

DEPENDENCY

@@ -0,0 +1,5 @@
+{MongoDB}
+{MongoDB C++ driver}
+Take a look at makefile (currently not configured at all)
+make
+Test with small .osm file

README

@@ -0,0 +1,118 @@
+osmo: Populate a MongoDB collection with OpenStreetMap data
+
+
+Features
+
+- osmo populates a MongoDB database collection with all nodes, ways and relations
+- Currently, only OSM XML (.osm) is supported
+- It's just slightly slower than Osmosis (with Postgres)
+- Prerequisites: see DEPENDENCY
+- Building:
+
+	make
+
+	(A priority is it to make this work in both Mac OS and Linux!)
+
+- Synopsis:
+
+	bin/osmo DATAFILE DATABASE.COLLECTION
+
+- To add an index on the primary key:
+
+	mongo DATABASE --eval \
+"printjson(db.COLLECTION.ensureIndex({ p: 1, id: 1 }, { unique: true, dropDups: true, background: true }))"
+
+
+Data model
+
+The data model supported by osmo is focussed on minimal data size and unification. Some principles:
+
+- nodes, ways and relations are transformed to GeoObject instances to unify the concept and to simplify queries on only 1 collection
+- Every geoObject has a key named "p" (prefix). It can have the values, "n" (node), "w" (way) or "r" (relation).
+- keys for regular fields (user, timestamp, ...) are abbreviated to save space
+- The geoObject primary key is [ 'p', 'id' ]
+- Example for a GeoObject instance:
+
+	{
+		"_id" : ObjectId("4e3213dbe9b522a655e3f5ff"),
+		"c" : 4977984,
+		"id" : 318214,
+		"p" : "r",
+		"parts" : { 
+			"list" : [
+				{
+					"p" : "n",
+					"id" : 319621296
+				},
+				{
+					"p" : "n",
+					"id" : 560090559
+				},
+				{
+					"p" : "n",
+					"id" : 292635574
+				},
+				{
+					"p" : "n",
+					"id" : 292635576
+				},
+				{
+					"p" : "n",
+					"id" : 245431191
+				},
+				{
+					"p" : "w",
+					"id" : 48289915
+				},
+				{
+					"p" : "w",
+					"id" : 49366289
+				},
+				...
+			],
+			"info" : [
+				{
+					"role" : "forward"
+				},
+				{
+					"role" : "forward"
+				},
+				{
+					"role" : ""
+				},
+				{
+					"role" : "forward"
+				},
+				{
+					"role" : ""
+				},
+				{
+					"role" : "forward"
+				},
+				{
+					"role" : "forward"
+				},
+				...
+			]
+		}, 
+		"t" : 1276427410,
+		"tags" : { 
+			"network" : "VRT",
+			"operator" : "SWT",
+			"ref" : "4",
+			"route" : "bus",
+			"type" : "route"
+		}, 
+		"u" : { 
+			"id" : 109925,
+			"name" : "WanMil"
+		}, 
+		"v" : 6
+	}
+
+- "parts" is an ObjectView object that has always two properties: "list" (a list of GeoObject references) and "info" (that optionally holds information about the referenced objects in the same order). GeoObjects with p=n do not have parts, with p=w, parts are always references to nodes, with p=r parts can reference any type (node, way, relation).
+
+
+Contribute
+
+Take a look at TODO to see a list of priorized features that are still lacking.

TODO

@@ -0,0 +1,16 @@
+Next steps
+
+Addd option parsing
+	connection data
+.osc support
+Validate data
+	Assert that necessary attributes for an element are set
+	...
+Debug & unify makefile (must work at least in Linux, MacOS)
+	configure?
+	I need help here
+
+
+Possible features
+
+Support .pbf

makefile

@@ -0,0 +1,9 @@
+CC = g++
+CFLAGS += -Wall -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -lmongoclient -lboost_thread -lboost_filesystem -lboost_program_options -lboost_system -lpthread -lexpat -I/Users/andi/.libraries/mongo
+
+bin/osmo:
+	$(CC) $(CFLAGS) src/osmo.cc -o bin/osmo
+
+.PHONY : clean
+clean:
+	rm bin/*

src/mongodb.h

@@ -0,0 +1,189 @@
+#include <string>
+#include <cstring>
+#include <sstream>
+#include <iostream>
+#include <unistd.h>
+#include <expat.h>
+#include "client/dbclient.h"
+
+using namespace mongo;
+using namespace osmo;
+
+
+
+
+namespace osmo {
+
+	namespace mongodb {
+
+		typedef struct {
+			DBClientConnection* c;
+			const char* collection;
+			GeoObject geoObject;
+	    } context;
+
+		const int BUFFER_SIZE = 10240 * 1;
+
+		void init_object (GeoObject* geoObject, const XML_Char* element, const XML_Char** attrs) {
+
+		    geoObject->reset();
+		    geoObject->set_attr("p", element);
+		    for (int i = 0; attrs[i]; i += 2) {
+
+		    		geoObject->set_attr(attrs[i], attrs[i + 1]);
+		    }
+		}
+
+		void XMLCALL startElement(void* data, const XML_Char* element, const XML_Char** attrs) {
+
+			context* ct = (context*)data;
+			GeoObject* geoObject = &ct->geoObject;
+
+		    // order in the following "if" statements is based on frequency of tags in planet file
+		    if (!strcmp(element, "nd")) {
+
+		    		GeoObjectRef geoObjectRef;
+		    		geoObjectRef.set_attr("p", "n");
+		        for (int i = 0; attrs[i]; i += 2) {
+		        
+		        		geoObjectRef.set_attr(attrs[i], attrs[i + 1]);
+		        }
+		        geoObject->parts->add(geoObjectRef);
+		    }
+		    else if (!strcmp(element, "node")) {
+
+		        init_object(geoObject, element, attrs);
+		    }
+		   	else if (!strcmp(element, "tag")) {
+
+				Tag tag;
+		        for (int i = 0; attrs[i]; i += 2) {
+		        
+		            tag.set_attr(attrs[i], attrs[i + 1]);
+		        }
+		        geoObject->add_tag(tag);
+		    }
+		    else if (!strcmp(element, "way")) {
+		    
+		        init_object(geoObject, element, attrs);
+		    }
+		    else if (!strcmp(element, "member")) {
+
+				GeoObjectRef geoObjectRef;
+				GeoObjectRel geoObjectRel;
+		        for (int i = 0; attrs[i]; i += 2) {
+		        
+		        		geoObjectRef.set_attr(attrs[i], attrs[i + 1]);
+		        		geoObjectRel.set_attr(attrs[i], attrs[i + 1]);
+		        }
+		        	geoObject->parts->add(geoObjectRef, geoObjectRel);
+		    }
+		    else if (!strcmp(element, "relation")) {
+		    
+				init_object(geoObject, element, attrs);
+		    }
+		}
+
+		void XMLCALL endElement(void *data, const XML_Char* element) {
+
+			context* ct = (context*)data;
+
+		    if (!strcmp(element, "node") || !strcmp(element, "way") || !strcmp(element, "relation")) {
+
+		        ct->c->insert(ct->collection, ct->geoObject.getBson());
+		        //std::cout << ct->c->getLastErrorDetailed().jsonString() << " ";
+		    }
+		}
+
+		void parse (int fd, DBClientConnection* c, const char* collection) {
+		
+		    bool done;
+		    register context ct = {
+		    		c,
+		    		collection,
+				GeoObject()
+		    };
+
+		    //c->dropCollection(collection);
+
+		    XML_Parser parser = XML_ParserCreate(0);
+		    if (!parser) {
+		        throw std::runtime_error("Error creating parser");
+		    }
+
+		    XML_SetUserData(parser, (void*)&ct);
+
+		    XML_SetElementHandler(parser, startElement, endElement);
+
+		    do {
+		        void *buffer = XML_GetBuffer(parser, BUFFER_SIZE);
+		        if (buffer == 0) {
+		            throw std::runtime_error("Out of memory");
+		        }
+
+		        int result = read(fd, buffer, BUFFER_SIZE);
+		        if (result < 0) {
+		            exit(1);
+		        }
+		        done = (result == 0);
+		        if (XML_ParseBuffer(parser, result, done) == XML_STATUS_ERROR)
+		        {
+		            XML_Error errorCode = XML_GetErrorCode(parser);
+		            long errorLine = XML_GetCurrentLineNumber(parser);
+		            long errorCol = XML_GetCurrentColumnNumber(parser);
+		            const XML_LChar *errorString = XML_ErrorString(errorCode);
+
+		            std::stringstream errorDesc;
+		            errorDesc << "XML parsing error at line " << errorLine << ":" << errorCol;
+		            errorDesc << ": " << errorString;
+		            throw std::runtime_error(errorDesc.str());
+		        }
+		    } while (!done);
+
+		    XML_ParserFree(parser);
+		    //error = "";
+		}
+		
+
+		void osm2db (const char* file, const char* collection) {
+
+			int fd = open(file, O_RDONLY);
+			if (fd < 0) {
+
+				std::cerr << "Can't open data file" << std::endl;
+				exit(1);
+			}
+
+			DBClientConnection* c = new DBClientConnection;
+			try {
+
+				c->connect("localhost");
+			} catch (DBException &e) {
+		
+				std::cerr << "Cannot connect: " << e.what() << std::endl;
+				exit(1);
+			}
+
+
+			char *suffix = strrchr(file, '.');
+			if (!strcmp(suffix, ".osm")) {
+
+				parse(fd, c, collection);
+			}
+			/*else if (!strcmp(suffix, ".osc")) {
+
+			}
+			else if (!strcmp(suffix, ".pbf")) {
+
+			}*/
+			else {
+
+				std::cerr << "Unknown file suffix: " << suffix << std::endl;
+				exit(1);
+			}
+
+
+			close(fd);
+		}
+	}
+}

src/osmo.cc

@@ -0,0 +1,34 @@
+#include <iostream>
+/*#include <cmath>*/
+#include "osmo.h"
+
+
+
+
+int main (int c, char *argv[]) {
+
+	if (c >= 3) {
+
+		osmo::mongodb::osm2db(argv[1], argv[2]);
+	}
+
+	/*
+	GeoObject* g = new GeoObject;
+	g->p = GeoObject::P_node;
+	g->id = 1000;
+	Tag* t = new Tag;
+	
+	memccpy(t->k, "amenity", 0, 255);
+	memccpy(t->v, "cafe", 0, 255);
+	g->add_tag(t);
+	g->add_tag(t);
+
+	g->parts->add(g);
+
+	std::cout << g->parts->refs[0]->id;
+	*/
+
+	std::cout << "ready";
+
+	return 0;
+}