sqlite: initial commit

Author: missinglink

.jshintrc

@@ -16,7 +16,7 @@
   "quotmark": "single",
   "undef": true,
   "unused": false,
-  "maxparams": 4,
+  "maxparams": 5,
   "maxdepth": 4,
   "maxlen": 140
 }

Placeholder.js

@@ -1,12 +1,10 @@
 
 var _ = require('lodash'),
-    TokenGraph = require('./lib/TokenGraph'),
     DocStore = require('./lib/DocStore');
 
 // constructor
-function Placeholder(){
-  this.graph = new TokenGraph();
-  this.store = new DocStore();
+function Placeholder( options ){
+  this.store = new DocStore( options );
 }
 
 // load prototype methods from modules

cmd/build.sh

@@ -4,7 +4,7 @@ DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd );
 
 PLACEHOLDER_DATA=${PLACEHOLDER_DATA:-"./data"};
 
-rm -f ${PLACEHOLDER_DATA}/graph.json ${PLACEHOLDER_DATA}/store.sqlite3;
+rm -f ${PLACEHOLDER_DATA}/store.sqlite3;
 
 cat ${PLACEHOLDER_DATA}/wof.extract | node ${DIR}/load.js
 

cmd/load.js

@@ -15,11 +15,11 @@ process.stdin.pipe( split() )
              .pipe( through.obj( function insert( row, _, next ){
                ph.insertWofRecord( row, next );
              }, function flush( next ){
-               ph.printStatistics();
-               console.error('sorting...');
-               ph.graph.sort(); // sort all arrays
-               console.error('vacuuming sqlite db...');
-               ph.store.db.run('VACUUM;');
+              //  ph.printStatistics();
+               console.error('pre commit...');
+               ph.store.preCommit();
+              //  console.error('sorting...');
+              //  ph.graph.sort(); // sort all arrays
                console.error('saving...');
                ph.save();
                next();

lib/DocStore.js

@@ -14,8 +14,14 @@ var codec = {
 // connect to and configure sqlite3 database
 function DocStore(){}
 
-DocStore.prototype.open = function( path ){
-  this.db = new sqlite3.Database( path );
+DocStore.prototype.open = function( path, options ){
+
+  if( options && true === options.readonly ){
+    this.db = new sqlite3.Database( path, sqlite3.OPEN_READONLY );
+  } else {
+    this.db = new sqlite3.Database( path );
+  }
+
   this.configure();
 };
 
@@ -37,8 +43,62 @@ DocStore.prototype.configure = function(){
 DocStore.prototype.reset = function(){
   this.db.serialize(function(){
     this.db.run('DROP TABLE IF EXISTS docs;');
+    this.db.run('DROP TABLE IF EXISTS lineage;');
+    this.db.run('DROP TABLE IF EXISTS tokens;');
+    this.db.run('DROP TABLE IF EXISTS fulltext;');
+    this.db.run('VACUUM;');
+
+    this.db.run('CREATE TABLE docs( id INTEGER PRIMARY KEY, json TEXT );');
+    this.db.run('CREATE TABLE lineage( id INTEGER, pid INTEGER );');
+    this.db.run('CREATE TABLE tokens( id INTEGER, lang STRING, tag STRING, token STRING );');
+
+    var options = [
+      'token',
+      'tokenize="' + [
+        'unicode61',
+        'remove_diacritics 0',
+        'tokenchars \'_\''
+      ].join(' ') + '"',
+      'prefix=1',
+      'prefix=2',
+      'prefix=3',
+      'prefix=4',
+      'prefix=5',
+      'prefix=6',
+      'prefix=7',
+      'prefix=8',
+      'prefix=9',
+      'prefix=10',
+      'prefix=11',
+      'prefix=12',
+      'prefix=13',
+      // 'content=\'\'',
+      // 'detail=none',
+      'columnsize=0'
+    ].join(', ');
+
+    this.db.run('CREATE VIRTUAL TABLE fulltext USING fts5(' + options + ');');
+  }.bind(this));
+};
+
+DocStore.prototype.preCommit = function(){
+  this.db.serialize(function(){
+
+    console.error('create indices...');
+    this.db.run('CREATE INDEX IF NOT EXISTS lineage_cover_idx ON lineage(id, pid);');
+    this.db.run('CREATE INDEX IF NOT EXISTS tokens_cover_idx ON tokens(id, lang, tag);');
+
+    // this is quite large, could use the fulltext index instead?
+    this.db.run('CREATE INDEX IF NOT EXISTS tokens_token_idx ON tokens(token);');
+
+    console.error('create fulltext table...');
+    this.db.run('INSERT INTO fulltext(rowid, token) SELECT rowid, REPLACE(token," ","_") FROM tokens;');
+
+    console.error('optimize fulltext table...');
+    this.db.run('INSERT INTO fulltext(fulltext) VALUES(\'optimize\');');
+
+    console.error('vacuuming sqlite db...');
     this.db.run('VACUUM;');
-    this.db.get('CREATE TABLE docs( id INTEGER PRIMARY KEY, json TEXT );');
   }.bind(this));
 };
 
@@ -49,6 +109,26 @@ DocStore.prototype.set = function( id, doc, cb ){
   );
 };
 
+DocStore.prototype.setLineage = function( id, pids, cb ){
+  if( !Array.isArray( pids ) || !pids.length ){ return cb(); }
+  this.db.run(
+    'INSERT INTO lineage ( id, pid ) VALUES ' + pids.map( pid => {
+      return '(' + id + ',' + pid + ')';
+    }).join(',') + ';',
+    cb
+  );
+};
+
+DocStore.prototype.setTokens = function( id, tokens, cb ){
+  if( !Array.isArray( tokens ) || !tokens.length ){ return cb(); }
+  this.db.run(
+    'INSERT INTO tokens ( id, lang, tag, token ) VALUES ' + tokens.map( token => {
+      return '(' + id + ',"' + token.lang + '","' + token.tag + '","' + token.body + '")';
+    }).join(',') + ';',
+    cb
+  );
+};
+
 DocStore.prototype.get = function( id, cb ){
   this.db.get(
     'SELECT json FROM docs WHERE id = $id LIMIT 1;',

package.json

@@ -31,6 +31,7 @@
   },
   "homepage": "https://github.com/pelias/placeholder#readme",
   "dependencies": {
+    "async": "^2.5.0",
     "express": "^4.15.2",
     "lodash": "^4.17.4",
     "lower-case": "^1.1.4",
@@ -40,7 +41,7 @@
     "require-dir": "^0.3.1",
     "sorted-intersect": "^0.1.4",
     "split2": "^2.1.1",
-    "sqlite3": "^3.1.8",
+    "sqlite3": "^3.1.13",
     "through2": "^2.0.3"
   },
   "devDependencies": {
@@ -50,7 +51,7 @@
     "precommit-hook": "^3.0.0",
     "semantic-release": "^8.2.0",
     "tap-spec": "^4.1.1",
-    "tape": "^4.6.3"
+    "tape": "^4.8.0"
   },
   "pre-commit": [
     "lint",

prototype/io.js

@@ -4,44 +4,39 @@ var fs = require('fs'),
     path = require('path');
 
 var dataDir = process.env.PLACEHOLDER_DATA || path.join( __dirname, '../data/');
-var graphPath = path.join( dataDir, 'graph.json' );
 var storePath = path.join( dataDir, 'store.sqlite3' );
 
+// WIP
+var SqlDatabase = require('../wip/SqlDatabase');
+
 // load data from disk
 module.exports.load = function( opts ){
   this.store.open( storePath );
   if( opts && opts.reset === true ){
     this.store.reset();
-  } else {
-    var graph = require( graphPath );
-    this.graph.nodes = graph.nodes;
-    this.graph.edges = graph.edges;
   }
+
+  // sql
+  var db = new SqlDatabase( this.store.db );
+
+  // both
+  var tokenize = require('../wip/test_tokenize').tokenize.bind({ db: db });
+  var query = require('../wip/query').query.bind( null, db, tokenize );
+
+  // WIP
+  this.wip = {
+    db: db,
+    tokenize: tokenize,
+    query: query
+  };
 };
 
 // save data to disk
 module.exports.save = function( path ){
-  fs.writeFileSync( graphPath, JSON.stringify( this.graph ) );
   this.close();
 };
 
 // gracefully close connections
 module.exports.close = function(){
   this.store.close();
 };
-
-// deserialize data
-// module.exports.import = function( data ){
-//   this.graph.nodes = data.nodes;
-//   this.graph.edges = data.edges;
-//   this.store.docs  = data.docs;
-// };
-//
-// // serialize data
-// module.exports.export = function( path ){
-//   return {
-//     docs:  this.store.docs,
-//     nodes: this.graph.nodes,
-//     edges: this.graph.edges
-//   };
-// };

prototype/wof.js

@@ -30,7 +30,9 @@ function insertWofRecord( wof, next ){
       lat: wof['lbl:latitude'] || wof['geom:latitude'],
       lon: wof['lbl:longitude'] ||wof['geom:longitude']
     },
-    names: {}
+    names: {},
+    tokens: [],
+    parentIds: []
   };
 
   // --- cast strings to numeric types ---
@@ -44,44 +46,41 @@ function insertWofRecord( wof, next ){
 
   // --- tokens ---
 
-  // convenience function with $id bound as first argument
-  var addToken = this.graph.addToken.bind( this.graph, id );
-
   // disable adding tokens to the index for the 'empire' placetype.
   // this ensures empire records are not retrieved via search.
   if( 'empire' !== doc.placetype ){
 
     // add 'wof:label'
-    analysis.normalize( wof['wof:label'] ).forEach( addToken );
+    doc.tokens.push({ lang: 'und', tag: 'label', body: wof['wof:label'] });
 
     // add 'wof:name'
-    analysis.normalize( wof['wof:name'] ).forEach( addToken );
+    doc.tokens.push({ lang: 'und', tag: 'label', body: wof['wof:name'] });
 
     // add 'wof:abbreviation'
-    analysis.normalize( wof['wof:abbreviation'] ).forEach( addToken );
+    doc.tokens.push({ lang: 'und', tag: 'abbr', body: wof['wof:abbreviation'] });
 
     // add 'ne:abbrev'
-    // analysis.normalize( wof['ne:abbrev'] ).forEach( addToken );
+    // doc.tokens.push({ lang: 'und', body: wof['ne:abbrev'] });
 
     // fields specific to countries & dependencies
     if( 'country' === doc.placetype || 'dependency' === doc.placetype ) {
       if( wof['iso:country'] && wof['iso:country'] !== 'XX' ){
 
         // add 'ne:iso_a2'
-        analysis.normalize( wof['ne:iso_a2'] ).forEach( addToken );
+        doc.tokens.push({ lang: 'und', tag: 'abbr', body: wof['ne:iso_a2'] });
 
         // add 'ne:iso_a3'
-        analysis.normalize( wof['ne:iso_a3'] ).forEach( addToken );
+        doc.tokens.push({ lang: 'und', tag: 'abbr', body: wof['ne:iso_a3'] });
 
         // add 'wof:country'
         // warning: eg. FR for 'French Guiana'
-        // analysis.normalize( wof['wof:country'] ).forEach( addToken );
+        // doc.tokens.push({ lang: 'und', tag: 'abbr', body: wof['wof:country'] });
 
         // add 'iso:country'
-        analysis.normalize( wof['iso:country'] ).forEach( addToken );
+        doc.tokens.push({ lang: 'und', tag: 'abbr', body: wof['iso:country'] });
 
         // add 'wof:country_alpha3'
-        analysis.normalize( wof['wof:country_alpha3'] ).forEach( addToken );
+        doc.tokens.push({ lang: 'und', tag: 'abbr', body: wof['wof:country_alpha3'] });
       }
     }
 
@@ -97,7 +96,11 @@ function insertWofRecord( wof, next ){
 
         // index each alternative name
         for( var n in wof[ attr ] ){
-          analysis.normalize( wof[ attr ][ n ] ).forEach( addToken );
+          doc.tokens.push({
+            lang: match[1],
+            tag: match[2],
+            body: wof[ attr ][ n ]
+          });
         }
 
         // doc - only store 'preferred' strings
@@ -121,7 +124,7 @@ function insertWofRecord( wof, next ){
     parentId = wof['wof:parent_id'];
     if( 'string' === typeof parentId ){ parentId = parseInt( parentId, 10 ); }
     if( !isNaN( parentId ) && parentId !== id && parentId > 0 ){
-      this.graph.setEdge( parentId, id ); // is child of
+      doc.parentIds.push( parentId ); // is child of
     }
   }
 
@@ -131,15 +134,62 @@ function insertWofRecord( wof, next ){
      var pid = wof['wof:hierarchy'][h][i];
      if( 'string' === typeof pid ){ pid = parseInt( pid, 10 ); }
      if( pid === id || pid <= 0 || pid === parentId ){ continue; }
-     //  this.graph.setEdge( id, pid, 'p' ); // has parent
-     this.graph.setEdge( pid, id ); // is child of
+     //  doc.parentIds.push( id, pid, 'p' ); // has parent
+     doc.parentIds.push( pid ); // is child of
    }
   }
 
+  // ---- consume aggregates
+
+  // normalize tokens
+  doc.tokens = doc.tokens.reduce(( res, token ) => {
+    analysis.normalize( token.body ).forEach( norm => {
+      res.push({ lang: token.lang, tag: token.tag, body: norm });
+    });
+    return res;
+  }, []);
+
+  // deduplicate tokens
+  var seen = {};
+  doc.tokens = doc.tokens.filter( token => {
+    return seen.hasOwnProperty( token.body ) ? false : ( seen[ token.body ] = true );
+  });
+
+  // store tokens in graph
+  // doc.tokens.forEach(token => {
+  //   this.graph.addToken( doc.id, token );
+  // }, this);
+
+  // deduplicate parent ids
+  doc.parentIds = doc.parentIds.filter(( pid, pos ) => {
+    return doc.parentIds.indexOf( pid ) === pos;
+  });
+
+  // store parent ids
+  // doc.parentIds.forEach(pid => {
+  //   this.graph.setEdge( pid, doc.id );
+  // }, this);
+
   // --- store ---
   // add doc to store
-  this.store.set( id, doc, next );
 
+  var tokens = doc.tokens;
+  var parentIds = doc.parentIds;
+
+  // --- delete fields
+  delete doc.tokens;
+  delete doc.parentIds;
+
+  this.store.set( id, doc, ( err ) => {
+    if( err ){ console.error( err ); }
+    this.store.setTokens( id, tokens, ( err ) => {
+      if( err ){ console.error( err ); }
+      this.store.setLineage( id, parentIds, ( err ) => {
+        if( err ){ console.error( err ); }
+        next();
+      });
+    });
+  });
 }
 
 // check if value is a valid number