diff --git a/controller/libpostal.js b/controller/libpostal.js index 7ecbe3b6e..48763e1ef 100644 --- a/controller/libpostal.js +++ b/controller/libpostal.js @@ -9,6 +9,7 @@ var field_mapping = { island: 'island', category: 'category', house: 'query', + unit: 'unit', house_number: 'number', road: 'street', suburb: 'neighbourhood', diff --git a/helper/diffPlaces.js b/helper/diffPlaces.js index 712c79595..9c28a935a 100644 --- a/helper/diffPlaces.js +++ b/helper/diffPlaces.js @@ -102,6 +102,11 @@ function assertAddressMatch(item1, item2) { propMatch(item1.address_parts, item2.address_parts, 'zip'); } + // only compare unit if both records have it, otherwise just ignore and assume it's the same + if (item1.address_parts.hasOwnProperty('unit') && item2.address_parts.hasOwnProperty('unit')) { + propMatch(item1.address_parts, item2.address_parts, 'unit'); + } + return false; } diff --git a/helper/geojsonify_place_details.js b/helper/geojsonify_place_details.js index c80da8b19..15c7650f6 100644 --- a/helper/geojsonify_place_details.js +++ b/helper/geojsonify_place_details.js @@ -6,6 +6,7 @@ const _ = require('lodash'); // If a property is identified as a single string, assume it should be presented as a string in response // If something other than string is desired, use the following structure: { name: 'category', type: 'array' } const DETAILS_PROPS = [ + { name: 'unit', type: 'string' }, { name: 'housenumber', type: 'string' }, { name: 'street', type: 'string' }, { name: 'postalcode', type: 'string' }, diff --git a/middleware/localNamingConventions.js b/middleware/localNamingConventions.js index 20c77f6b3..bfdd7691d 100644 --- a/middleware/localNamingConventions.js +++ b/middleware/localNamingConventions.js @@ -45,8 +45,18 @@ function applyLocalNamingConventions(req, res, next) { // flip the housenumber and street name // eg. '101 Grolmanstraße' -> 'Grolmanstraße 101' function flipNumberAndStreet(place) { - var standard = ( place.address_parts.number + ' ' + place.address_parts.street ), - flipped = ( place.address_parts.street + ' ' + place.address_parts.number ); + var standard = ( [ place.address_parts.number, place.address_parts.street ] ), + flipped = ( [ place.address_parts.street, place.address_parts.number ] ); + + // unit attribte added if set + if(place.address_parts.hasOwnProperty('unit')) { + standard.push(place.address_parts.unit); + flipped.push(place.address_parts.unit); + } + + // join into strings + standard = standard.join(' '); + flipped = flipped.join(' '); // flip street name and housenumber if( place.name.default === standard ){ diff --git a/middleware/renamePlacenames.js b/middleware/renamePlacenames.js index 9f63cb6e3..b56a6139b 100644 --- a/middleware/renamePlacenames.js +++ b/middleware/renamePlacenames.js @@ -5,6 +5,7 @@ const _ = require('lodash'); const PARENT_PROPS = require('../helper/placeTypes'); const ADDRESS_PROPS = [ + { name: 'unit', newName: 'unit' }, { name: 'number', newName: 'housenumber' }, { name: 'zip', newName: 'postalcode', transform: (value) => { return [value]; } }, { name: 'street', newName: 'street' } diff --git a/package.json b/package.json index d32e37cc3..154717ea3 100644 --- a/package.json +++ b/package.json @@ -59,8 +59,8 @@ "pelias-labels": "1.7.0", "pelias-logger": "0.3.0", "pelias-microservice-wrapper": "1.3.0", - "pelias-model": "5.2.0", - "pelias-query": "9.1.1", + "pelias-model": "sweco-semhul/model#adding-address-unit", + "pelias-query": "sweco-semhul/query#adding-address-unit", "pelias-sorting": "1.1.0", "predicates": "^2.0.0", "retry": "^0.10.1", diff --git a/query/autocomplete_defaults.js b/query/autocomplete_defaults.js index 34f396aeb..b90d547d9 100644 --- a/query/autocomplete_defaults.js +++ b/query/autocomplete_defaults.js @@ -36,6 +36,10 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', + 'address:unit:analyzer': 'peliasUnit', + 'address:unit:field': 'address_parts.unit', + 'address:unit:boost': 2, + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, diff --git a/query/reverse_defaults.js b/query/reverse_defaults.js index bce720574..bb3aa9bb6 100644 --- a/query/reverse_defaults.js +++ b/query/reverse_defaults.js @@ -37,6 +37,10 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', + 'address:unit:analyzer': 'peliasUnit', + 'address:unit:field': 'address_parts.unit', + 'address:unit:boost': 2, + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, diff --git a/query/search_defaults.js b/query/search_defaults.js index c0d2a6a0f..c6fa8b216 100644 --- a/query/search_defaults.js +++ b/query/search_defaults.js @@ -36,6 +36,10 @@ module.exports = _.merge({}, peliasQuery.defaults, { 'function_score:score_mode': 'avg', 'function_score:boost_mode': 'replace', + 'address:unit:analyzer': 'peliasUnit', + 'address:unit:field': 'address_parts.unit', + 'address:unit:boost': 2, + 'address:housenumber:analyzer': 'peliasHousenumber', 'address:housenumber:field': 'address_parts.number', 'address:housenumber:boost': 2, diff --git a/query/search_original.js b/query/search_original.js index a6b666d6a..0be404808 100644 --- a/query/search_original.js +++ b/query/search_original.js @@ -30,6 +30,7 @@ query.score( peliasQuery.view.popularity( peliasQuery.view.phrase ) ); query.score( peliasQuery.view.population( peliasQuery.view.phrase ) ); // address components +query.score( peliasQuery.view.address('unit') ); query.score( peliasQuery.view.address('housenumber') ); query.score( peliasQuery.view.address('street') ); query.score( peliasQuery.view.address('postcode') ); diff --git a/query/text_parser.js b/query/text_parser.js index fd07cd0d9..8db935022 100644 --- a/query/text_parser.js +++ b/query/text_parser.js @@ -19,6 +19,11 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){ vs.var( 'input:address', parsed_text.address ); } + // unit - an apartment, unit, office, lot, or other secondary unit designator + if( ! _.isEmpty(parsed_text.unit) ){ + vs.var( 'input:unit', parsed_text.unit ); + } + // house number if( ! _.isEmpty(parsed_text.number) ){ vs.var( 'input:housenumber', parsed_text.number ); diff --git a/query/text_parser_addressit.js b/query/text_parser_addressit.js index 84979c15e..2359416ed 100644 --- a/query/text_parser_addressit.js +++ b/query/text_parser_addressit.js @@ -36,6 +36,11 @@ function addParsedVariablesToQueryVariables( parsed_text, vs ){ // ==== add parsed matches [address components] ==== + // unit + if( parsed_text.hasOwnProperty('unit') ){ + vs.var( 'input:unit', parsed_text.unit ); + } + // house number if( parsed_text.hasOwnProperty('number') ){ vs.var( 'input:housenumber', parsed_text.number ); diff --git a/sanitizer/_text_addressit.js b/sanitizer/_text_addressit.js index 6cf31f4d3..2da75c064 100644 --- a/sanitizer/_text_addressit.js +++ b/sanitizer/_text_addressit.js @@ -85,6 +85,7 @@ function parse(query) { _.extend(addressWithAdminParts, addressWithAddressParts); var address_parts = [ 'name', + 'unit', 'number', 'street', 'city', diff --git a/service/configurations/Interpolation.js b/service/configurations/Interpolation.js index 1bb08d779..fb3ff028e 100644 --- a/service/configurations/Interpolation.js +++ b/service/configurations/Interpolation.js @@ -12,12 +12,16 @@ class Language extends ServiceConfiguration { } getParameters(req, hit) { - return { + let res = { number: req.clean.parsed_text.number, street: hit.address_parts.street || req.clean.parsed_text.street, lat: hit.center_point.lat, lon: hit.center_point.lon }; + if(req.clean.parsed_text.hasOwnProperty('unit')) { + res.unit = req.clean.parsed_text.unit; + } + return res; } diff --git a/test/unit/controller/libpostal.js b/test/unit/controller/libpostal.js index cf62fff50..00a30166c 100644 --- a/test/unit/controller/libpostal.js +++ b/test/unit/controller/libpostal.js @@ -185,6 +185,10 @@ module.exports.tests.success_conditions = (test, common) => { label: 'house', value: 'house value' }, + { + label: 'unit', + value: 'unit value' + }, { label: 'house_number', value: 'house_number value' @@ -244,6 +248,7 @@ module.exports.tests.success_conditions = (test, common) => { island: 'island value', category: 'category value', query: 'house value', + unit: 'unit value', number: 'house_number value', street: 'road value', neighbourhood: 'suburb value', diff --git a/test/unit/fixture/search_full_address_original_with_unit.js b/test/unit/fixture/search_full_address_original_with_unit.js new file mode 100644 index 000000000..679b23036 --- /dev/null +++ b/test/unit/fixture/search_full_address_original_with_unit.js @@ -0,0 +1,161 @@ +var vs = require('../../../query/search_defaults'); + +module.exports = { + 'query': { + 'bool': { + 'must': [ + { + 'match': { + 'name.default': { + 'analyzer': 'peliasQueryFullToken', + 'boost': 1, + 'query': '431 St Kilda Rd' + } + } + } + ], + 'should': [ + { + 'match': { + 'phrase.default': { + 'analyzer': 'peliasPhrase', + 'type': 'phrase', + 'boost': 1, + 'slop': 2, + 'query': '431 St Kilda Rd' + } + } + }, + { + 'function_score': { + 'query': { + 'match': { + 'phrase.default': { + 'analyzer': 'peliasPhrase', + 'type': 'phrase', + 'boost': 1, + 'slop': 2, + 'query': '431 St Kilda Rd' + } + } + }, + 'max_boost': 20, + 'functions': [ + { + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'popularity', + 'missing': 1 + }, + 'weight': 1 + } + ], + 'score_mode': 'first', + 'boost_mode': 'replace' + } + }, + { + 'function_score': { + 'query': { + 'match': { + 'phrase.default': { + 'analyzer': 'peliasPhrase', + 'type': 'phrase', + 'boost': 1, + 'slop': 2, + 'query': '431 St Kilda Rd' + } + } + }, + 'max_boost': 20, + 'functions': [ + { + 'field_value_factor': { + 'modifier': 'log1p', + 'field': 'population', + 'missing': 1 + }, + 'weight': 2 + } + ], + 'score_mode': 'first', + 'boost_mode': 'replace' + } + }, + { + 'match': { + 'address_parts.unit': { + 'analyzer': vs['address:unit:analyzer'], + 'boost': vs['address:unit:boost'], + 'query': '8' + } + } + }, + { + 'match': { + 'address_parts.number': { + 'analyzer': vs['address:housenumber:analyzer'], + 'boost': vs['address:housenumber:boost'], + 'query': '431' + } + } + }, + { + 'match': { + 'address_parts.street': { + 'analyzer': vs['address:street:analyzer'], + 'boost': vs['address:street:boost'], + 'query': 'St Kilda Rd' + } + } + }, + { + 'match': { + 'parent.region_a': { + 'analyzer': vs['admin:region_a:analyzer'], + 'boost': vs['admin:region_a:boost'], + 'query': 'Melbourne' + } + } + }, + { + 'multi_match': { + 'fields': [ + 'parent.country^1', + 'parent.region^1', + 'parent.county^1', + 'parent.localadmin^1', + 'parent.locality^1', + 'parent.borough^1', + 'parent.neighbourhood^1', + 'parent.region_a^1' + ], + 'query': 'Melbourne', + 'analyzer': 'peliasAdmin' + } + } + ], + 'filter': [ + { + 'terms': { + 'layer': [ + 'address', + 'venue', + 'country', + 'region', + 'county', + 'neighbourhood', + 'locality', + 'localadmin' + ] + } + } + ] + } + }, + 'size': 10, + 'track_scores': true, + 'sort': [ + '_score' + ] +}; diff --git a/test/unit/helper/diffPlaces.js b/test/unit/helper/diffPlaces.js index a7dd692d7..da93965fc 100644 --- a/test/unit/helper/diffPlaces.js +++ b/test/unit/helper/diffPlaces.js @@ -166,6 +166,50 @@ module.exports.tests.dedupe = function(test, common) { t.false(isDifferent(item1, item2), 'should be the same'); t.end(); }); + + test('catch diff address by unit', function(t) { + var item1 = { + 'address_parts': { + 'unit': 'A', + 'number': '1', + 'street': 'Main Street', + 'zip': '90210' + } + }; + var item2 = { + 'address_parts': { + 'unit': 'B', + 'number': '1', + 'street': 'Main Street', + 'zip': '90210' + } + }; + + t.true(isDifferent(item1, item2), 'should be different'); + t.end(); + }); + + test('catch diff address by unit', function(t) { + var item1 = { + 'address_parts': { + 'unit': 'A', + 'number': '1', + 'street': 'Main Street', + 'zip': '90210' + } + }; + var item2 = { + 'address_parts': { + 'unit': 'A', + 'number': '1', + 'street': 'Main Street', + 'zip': '90210' + } + }; + + t.false(isDifferent(item1, item2), 'should be the same'); + t.end(); + }); }; module.exports.all = function (tape, common) { diff --git a/test/unit/helper/geojsonify_place_details.js b/test/unit/helper/geojsonify_place_details.js index c7f2c8ace..e0788574d 100644 --- a/test/unit/helper/geojsonify_place_details.js +++ b/test/unit/helper/geojsonify_place_details.js @@ -5,6 +5,7 @@ module.exports.tests = {}; module.exports.tests.geojsonify_place_details = (test, common) => { test('plain old string values should be copied verbatim, replacing old values', t => { const source = { + unit: 'unit value', housenumber: 'housenumber value', street: 'street value', postalcode: 'postalcode value', @@ -53,6 +54,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { }; const expected = { + unit: 'unit value', housenumber: 'housenumber value', street: 'street value', postalcode: 'postalcode value', @@ -110,6 +112,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { test('\'empty\' string-type values should be output as \'\'', t => { [ [], {}, '', 17, true, null, undefined ].forEach(empty_value => { const source = { + unit: empty_value, housenumber: empty_value, street: empty_value, postalcode: empty_value, @@ -170,6 +173,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { test('source arrays should be copy first value', t => { const source = { + unit: ['unit value 1', 'unit value 2'], housenumber: ['housenumber value 1', 'housenumber value 2'], street: ['street value 1', 'street value 2'], postalcode: ['postalcode value 1', 'postalcode value 2'], @@ -218,6 +222,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { }; const expected = { + unit: 'unit value 1', housenumber: 'housenumber value 1', street: 'street value 1', postalcode: 'postalcode value 1', @@ -275,6 +280,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { test('non-empty objects should be converted to strings', t => { // THIS TEST SHOWS THAT THE CODE DOES NOT DO WHAT IT EXPECTED const source = { + unit: { unit: 'unit value' }, housenumber: { housenumber: 'housenumber value'}, street: { street: 'street value'}, postalcode: { postalcode: 'postalcode value'}, @@ -323,6 +329,7 @@ module.exports.tests.geojsonify_place_details = (test, common) => { }; const expected = { + unit: '[object Object]', housenumber: '[object Object]', street: '[object Object]', postalcode: '[object Object]', diff --git a/test/unit/middleware/localNamingConventions.js b/test/unit/middleware/localNamingConventions.js index 1d4102f0e..028c4182b 100644 --- a/test/unit/middleware/localNamingConventions.js +++ b/test/unit/middleware/localNamingConventions.js @@ -90,8 +90,28 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { } }; + var dkAddressWithUnit = { + '_id': 'test5', + '_type': 'test', + 'name': { 'default': '26 Nikolaj Plads 2 th' }, + 'center_point': { 'lon': 12.580921, 'lat': 55.678665 }, + 'address_parts': { + 'zip': '1067', + 'unit': '2 th', + 'number': '26', + 'street': 'Nikolaj Plads' + }, + 'parent': { + 'region': ['København'], + 'locality': ['København K'], + 'country_a': ['DNK'], + 'county': ['Region Hovedstaden'], + 'country': ['Denmark'] + } + }; + var req = {}, - res = { data: [ ukAddress, deAddress, nlAddress, unknownCountryAddress ] }, + res = { data: [ ukAddress, deAddress, nlAddress, unknownCountryAddress, dkAddressWithUnit ] }, middleware = localNamingConventions(); test('flipNumberAndStreet', function(t) { @@ -113,6 +133,11 @@ module.exports.tests.flipNumberAndStreet = function(test, common) { // being disabled), don't have the name flipped t.equal( res.data[3].name.default, '123 Main Street', 'standard name'); + // DNK address should have the housenumber and street name flipped, too + // this definition comes from pelias configuration + // In this case address have unit and it should also be added to flipped address name + t.equal( res.data[4].name.default, 'Nikolaj Plads 26 2 th', 'flipped name' ); + t.end(); }); }); diff --git a/test/unit/query/search_original.js b/test/unit/query/search_original.js index 5e035679e..c0dd1b53f 100644 --- a/test/unit/query/search_original.js +++ b/test/unit/query/search_original.js @@ -113,6 +113,29 @@ module.exports.tests.query = function(test, common) { t.end(); }); + test('valid query with a full valid address with unit', function(t) { + var query = generate({ text: 'Shop 8, 431 St Kilda Rd Melbourne', + layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], + querySize: 10, + parsed_text: { + unit: '8', + number: '431', + street: 'St Kilda Rd', + regions: [ 'Melbourne' ] + } + }); + + var compiled = JSON.parse( JSON.stringify( query ) ); + + var expected = require('../fixture/search_full_address_original_with_unit'); + console.log(JSON.stringify( expected , null, 2)); + console.log('----'); + console.log(JSON.stringify( compiled.body , null, 2)); + t.deepEqual(compiled.type, 'original', 'query type set'); + t.deepEqual(compiled.body, expected, 'search_full_address'); + t.end(); + }); + test('valid query with partial address', function(t) { var query = generate({ text: 'soho grand, new york', layers: [ 'address', 'venue', 'country', 'region', 'county', 'neighbourhood', 'locality', 'localadmin' ], diff --git a/test/unit/query/text_parser.js b/test/unit/query/text_parser.js index 86d443fc0..2b641818f 100644 --- a/test/unit/query/text_parser.js +++ b/test/unit/query/text_parser.js @@ -19,6 +19,7 @@ module.exports.tests.query = function(test, common) { t.false(vs.isset('input:query')); t.false(vs.isset('input:category')); + t.false(vs.isset('input:unit')); t.false(vs.isset('input:housenumber')); t.false(vs.isset('input:street')); t.false(vs.isset('input:address')); @@ -37,6 +38,7 @@ module.exports.tests.query = function(test, common) { var parsed_text = { query: 'query value', category: 'category value', + unit: 'unit value', number: 'number value', street: 'street value', address: 'address value', @@ -54,6 +56,7 @@ module.exports.tests.query = function(test, common) { t.equals(vs.var('input:query').toString(), 'query value'); t.equals(vs.var('input:category').toString(), 'category value'); + t.equals(vs.var('input:unit').toString(), 'unit value'); t.equals(vs.var('input:housenumber').toString(), 'number value'); t.equals(vs.var('input:street').toString(), 'street value'); t.equals(vs.var('input:address').toString(), 'address value'); @@ -162,6 +165,7 @@ module.exports.tests.empty_values = function(test, common) { var parsed_text = { query: '', category: '', + unit: '', number: '', street: '', address: '', @@ -183,6 +187,7 @@ module.exports.tests.empty_values = function(test, common) { t.false(vs.isset('input:query')); t.false(vs.isset('input:category')); + t.false(vs.isset('input:unit')); t.false(vs.isset('input:housenumber')); t.false(vs.isset('input:street')); t.false(vs.isset('input:address')); diff --git a/test/unit/sanitizer/_text_addressit.js b/test/unit/sanitizer/_text_addressit.js index 5fad89a8a..b8d4a4b3a 100644 --- a/test/unit/sanitizer/_text_addressit.js +++ b/test/unit/sanitizer/_text_addressit.js @@ -236,6 +236,33 @@ module.exports.tests.text_parser = function(test, common) { }); + test('valid address, unit', function(t) { + var raw = { + text: 'Shop 8, 431 St Kilda Rd Melbourne' + }; + var clean = {}; + + var expected_clean = { + text: 'Shop 8, 431 St Kilda Rd Melbourne', + parser: 'addressit', + parsed_text: { + name: 'Shop 8', + unit: '8', + number: '431', + street: 'St Kilda Rd', + regions: [ 'Melbourne' ], + admin_parts: '431 St Kilda Rd Melbourne' + } + }; + + var messages = sanitizer.sanitize(raw, clean); + + t.deepEqual(messages, { errors: [], warnings: [] } ); + t.deepEqual(clean, expected_clean); + t.end(); + + }); + test('valid address, house number', function(t) { var raw = { text: '123 main st new york ny' diff --git a/test/unit/service/configurations/Interpolation.js b/test/unit/service/configurations/Interpolation.js index c71df69e9..40ec21363 100644 --- a/test/unit/service/configurations/Interpolation.js +++ b/test/unit/service/configurations/Interpolation.js @@ -115,6 +115,43 @@ module.exports.tests.all = (test, common) => { }); + test('getParameters should return unit when req.clean.parsed_text.unit is set', t => { + const configBlob = { + url: 'base url' + }; + + const interpolation = new Interpolation(configBlob); + + const req = { + clean: { + parsed_text: { + unit: 'parsed unit value', + number: 'parsed number value', + street: 'parsed street value' + } + } + }; + + const hit = { + address_parts: { + }, + center_point: { + lat: 12.121212, + lon: 21.212121 + } + }; + + t.deepEquals(interpolation.getParameters(req, hit), { + unit: 'parsed unit value', + number: 'parsed number value', + street: 'parsed street value', + lat: 12.121212, + lon: 21.212121 + }); + t.end(); + + }); + test('baseUrl ending in / should not have double /\'s return by getUrl', t => { const configBlob = { url: 'http://localhost:1234/'