feat(addendum): allow arbitrary data to be stored alongside the rest …

…of the document

Document.js

@@ -1,8 +1,8 @@
-var config = require('pelias-config').generate();
-
-var validate = require('./util/valid');
-var transform = require('./util/transform');
-var _ = require('lodash');
+const config = require('pelias-config').generate();
+const validate = require('./util/valid');
+const transform = require('./util/transform');
+const _ = require('lodash');
+const codec = require('./codec');
 
 const addressFields = ['name', 'number', 'unit', 'street', 'zip'];
 
@@ -32,6 +32,7 @@ function Document( source, layer, source_id ){
   this.address_parts = {};
   this.center_point = {};
   this.category = [];
+  this.addendum = {};
 
   // create a non-enumerable property for metadata
   Object.defineProperty( this, '_meta', { writable: true, value: {} });
@@ -70,9 +71,15 @@ Document.prototype.toESDocument = function() {
     bounding_box: this.bounding_box,
     popularity: this.popularity,
     population: this.population,
+    addendum: {},
     polygon: this.shape
   };
 
+  // add encoded addendum namespaces
+  for( var namespace in this.addendum || {} ){
+    doc.addendum[namespace] = codec.encode(this.addendum[namespace]);
+  }
+
   // remove empty properties
   if( !Object.keys( doc.parent || {} ).length ){
     delete doc.parent;
@@ -95,6 +102,9 @@ Document.prototype.toESDocument = function() {
   if (!this.popularity) {
     delete doc.popularity;
   }
+  if( !Object.keys( doc.addendum || {} ).length ){
+    delete doc.addendum;
+  }
   if( !Object.keys( doc.polygon || {} ).length ){
     delete doc.polygon;
   }
@@ -490,6 +500,33 @@ Document.prototype.removeCategory = function( value ){
   return this;
 };
 
+// addendum
+Document.prototype.setAddendum = function( namespace, value ){
+  validate.type('string', namespace);
+  validate.truthy(namespace);
+  validate.type('object', value);
+  if( Object.keys(value).length > 0 ){
+    this.addendum[ namespace ] = value;
+  }
+  return this;
+};
+
+Document.prototype.getAddendum = function( namespace ){
+  return this.addendum[ namespace ];
+};
+
+Document.prototype.hasAddendum = function( namespace ){
+  return this.addendum.hasOwnProperty( namespace );
+};
+
+Document.prototype.delAddendum = function( namespace ){
+  if( this.hasAddendum( namespace ) ){
+    delete this.addendum[ namespace ];
+    return true;
+  }
+  return false;
+};
+
 // centroid
 Document.prototype.setCentroid = function( centroid ){
   centroid = centroid || {};

README.md

@@ -39,6 +39,8 @@ var poi = new Document( 'geoname', 'venue', 1003 )
   .removeCategory( 'foo' )
   .setPopulation(10)
   .setPopularity(3)
+  .setAddendum('wikipedia', { slug: 'HackneyCityFarm' })
+  .setAddendum('geonames', { foreignkey: 1 })
   .setCentroid({ lon: 0.5, lat: 50.1 })
   .setPolygon( geojsonObject /* any valid geojson object */ )
   .setBoundingBox( bboxObject /* see tests for bbox syntax */ );

codec.js

@@ -0,0 +1,14 @@
+
+// Select the codec used for functionality which requires
+// encoding/decoding data before being sent to elasticsearch.
+
+// JSON codec
+const json = {
+  encode: (value) => JSON.stringify(value),
+  decode: (text)  => JSON.parse(text)
+};
+
+// msgpack codec
+// const msgpack = require("msgpack-lite");
+
+module.exports = json;

index.js

@@ -1,2 +1,3 @@
 module.exports.Document = require('./Document');
+module.exports.codec = require('./codec');
 module.exports.createDocumentMapperStream = require('./DocumentMapperStream');

test/document/addendum.js

@@ -0,0 +1,77 @@
+const Document = require('../../Document');
+module.exports.tests = {};
+
+const fixture = {
+  wikipedia: { slug: 'Wikipedia', population: 100 },
+  custom: { example: [ 'Example' ] }
+};
+
+module.exports.tests.getAddendum = function(test) {
+  test('getAddendum', function(t) {
+    var doc = new Document('mysource','mylayer','myid');
+    t.equal(doc.getAddendum('wikipedia'), undefined, 'getter works');
+    doc.addendum = fixture;
+    t.equal(doc.getAddendum('wikipedia'), fixture.wikipedia, 'getter works');
+    t.end();
+  });
+};
+
+module.exports.tests.setAddendum = function(test) {
+  test('setAddendum', function(t) {
+    var doc = new Document('mysource','mylayer','myid');
+    t.equal(doc.setAddendum('wikipedia',fixture.wikipedia), doc, 'chainable');
+    t.equal(doc.addendum.wikipedia, fixture.wikipedia, 'setter works');
+    t.end();
+  });
+  test('setAddendum - validate namespace', function(t) {
+    var doc = new Document('mysource','mylayer','myid');
+    t.throws( doc.setAddendum.bind(doc,1,fixture), null, 'invalid type' );
+    t.throws( doc.setAddendum.bind(doc,'',fixture), null, 'invalid length' );
+    t.throws( doc.setAddendum.bind(doc,' ',fixture), null, 'invalid length' );
+    t.throws( doc.setAddendum.bind(doc,null,fixture), null, 'invalid length' );
+    t.equal(doc.getAddendum('test'), undefined, 'property not set');
+    t.end();
+  });
+  test('setAddendum - validate val', function(t) {
+    var doc = new Document('mysource','mylayer','myid');
+    t.throws( doc.setAddendum.bind(doc,'wikipedia',1), null, 'invalid value' );
+    t.throws( doc.setAddendum.bind(doc,'wikipedia',''), null, 'invalid value' );
+    t.throws( doc.setAddendum.bind(doc,'wikipedia',' '), null, 'invalid value' );
+    t.throws( doc.setAddendum.bind(doc,'wikipedia',null), null, 'invalid value' );
+    t.throws( doc.setAddendum.bind(doc,'wikipedia','\t'), null, 'invalid value' );
+    t.equal(doc.getAddendum('test'), undefined, 'property not set');
+    t.end();
+  });
+};
+
+module.exports.tests.hasAddendum = function(test) {
+  test('hasAddendum', function(t) {
+    var doc = new Document('mysource','mylayer','myid');
+    t.equal(doc.hasAddendum('wikipedia'), false, 'hasser works');
+    doc.addendum = fixture;
+    t.equal(doc.hasAddendum('wikipedia'), true, 'hasser works');
+    t.end();
+  });
+};
+
+module.exports.tests.delAddendum = function(test) {
+  test('delAddendum', function(t) {
+    var doc = new Document('mysource','mylayer','myid');
+    t.equal(doc.delAddendum('wikipedia'), false, 'deller works');
+    doc.addendum = fixture;
+    t.equal(doc.delAddendum('wikipedia'), true, 'deller works');
+    t.equal(doc.addendum.wikipedia, undefined, 'deller works');
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('addendum: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

test/document/toESDocument.js

@@ -1,12 +1,13 @@
-var proxyquire = require('proxyquire');
+const proxyquire = require('proxyquire');
+const codec = require('../../codec');
 
 var fakeGeneratedConfig = {
   schema: {
     indexName: 'pelias'
   }
 };
 
-var fakeConfig = {
+const fakeConfig = {
   generate: function fakeGenerate() {
     return fakeGeneratedConfig;
   }
@@ -41,6 +42,8 @@ module.exports.tests.toESDocument = function(test) {
     doc.setPolygon({ key: 'value' });
     doc.addCategory('category 1');
     doc.addCategory('category 2');
+    doc.setAddendum('wikipedia', { slug: 'HackneyCityFarm' });
+    doc.setAddendum('geonames', { foreignkey: 1 });
 
     var esDoc = doc.toESDocument();
 
@@ -74,7 +77,11 @@ module.exports.tests.toESDocument = function(test) {
         category: [
           'category 1',
           'category 2'
-        ]
+        ],
+        addendum: {
+          wikipedia: codec.encode({ slug: 'HackneyCityFarm' }),
+          geonames: codec.encode({ foreignkey: 1 })
+        }
       }
     };
 

test/run.js

@@ -11,6 +11,7 @@ var tests = [
   require('./document/popularity.js'),
   require('./document/population.js'),
   require('./document/name.js'),
+  require('./document/addendum.js'),
   require('./document/address.js'),
   require('./document/parent.js'),
   require('./document/polygon.js'),

test/serialize/test.js

@@ -32,7 +32,8 @@ module.exports.tests.minimal = function(test) {
       'parent': {},
       'address_parts': {},
       'category': [],
-      'center_point': {}
+      'center_point': {},
+      'addendum': {}
     }, 'valid document body');
 
     t.end();
@@ -57,6 +58,8 @@ module.exports.tests.complete = function(test) {
       .removeCategory( 'foo' )
       .setPopulation(10)
       .setPopularity(3)
+      .setAddendum('wikipedia', { slug: 'HackneyCityFarm' })
+      .setAddendum('geonames', { foreignkey: 1 })
       .setCentroid({ lon: 0.5, lat: 50.1 })
       .setPolygon(fixtures.new_zealand)
       .setBoundingBox(fixtures.new_zealand_bbox);
@@ -128,7 +131,13 @@ module.exports.tests.complete = function(test) {
       'source_id': '1003',
       'category':['bar'],
       'population': 10,
-      'popularity': 3
+      'popularity': 3,
+
+      // addendum
+      'addendum': {
+        'wikipedia': { 'slug': 'HackneyCityFarm' },
+        'geonames': { 'foreignkey': 1 }
+      }
 
     }, 'valid document body');