Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding unit attribute to document stream to include addresses with th… #320

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Adding unit attribute to document stream to include addresses with th… #320

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e115434
Adding unit attribute to document stream to include addresses with th…
sweco-semhul Nov 2, 2017
f9b6165
Merge branch 'master' into adding-address-unit
Nov 3, 2017
16b8903
Merge branch 'master' into adding-address-unit
Nov 3, 2017
4dba4bc
Switching order of street and unit for default name
sweco-semhul Nov 20, 2017
968b0e6
Merge branch 'adding-address-unit' of https://github.com/sweco-semhul…
sweco-semhul Nov 20, 2017
973c247
Changing default name concatination to be a bit cleaner using array.join
sweco-semhul Nov 20, 2017
a756bd5
Adding space
sweco-semhul Nov 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion lib/streams/documentStream.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,21 @@ function createDocumentStream(id_prefix, stats) {
uid++;

try {
let name;
if (record.UNIT) {
name = record.NUMBER + ' ' + record.UNIT + ' ' + record.STREET;
} else {
name = record.NUMBER + ' ' + record.STREET;
}

const addrDoc = new peliasModel.Document( 'openaddresses', 'address', model_id )
.setName( 'default', (record.NUMBER + ' ' + record.STREET) )
.setName( 'default', name )
.setCentroid( { lon: record.LON, lat: record.LAT } );

if (record.UNIT) {
addrDoc.setAddress( 'unit', record.UNIT );
}

addrDoc.setAddress( 'number', record.NUMBER );

addrDoc.setAddress( 'street', record.STREET );
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"pelias-dbclient": "2.2.5",
"pelias-config": "2.13.0",
"pelias-logger": "0.3.0",
"pelias-model": "5.2.0",
"pelias-model": "sweco-semhul/model#adding-address-unit",
"pelias-wof-admin-lookup": "4.1.2",
"through2": "^2.0.3",
"through2-filter": "^2.0.0",
Expand Down
177 changes: 142 additions & 35 deletions test/data/expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,36 +10,116 @@
"phrase": {
"default": "100 Main St"
},
"address_parts": {
"number": "100",
"street": "Main St",
"zip": "input postcode"
},
"center_point": {
"lon": 21.111111,
"lat": 12.111111
},
"source": "openaddresses",
"layer": "address",
"source_id": "data/input_file_1:0"
}
},
{
"_index": "pelias",
"_type": "address",
"_id": "data/input_file_1:1",
"data": {
"name": {
"default": "100 1 Main St"
},
"phrase": {
"default": "100 1 Main St"
},
"parent": {
"country": [ "override country" ],
"country_id": [ "1" ],
"country_a": [ null ],
"macroregion": [ "override macroregion" ],
"macroregion_id": [ "2" ],
"macroregion_a": [ null ],
"region": [ "override region" ],
"region_id": [ "3" ],
"region_a": [ null ],
"macrocounty": [ "override macrocounty" ],
"macrocounty_id": [ "4" ],
"macrocounty_a": [ null ],
"county": [ "override county" ],
"county_id": [ "5" ],
"county_a": [ null ],
"borough": [ "override borough" ],
"borough_id": [ "6" ],
"borough_a": [ null ],
"locality": [ "override locality" ],
"locality_id": [ "7" ],
"locality_a": [ null ],
"localadmin": [ "override localadmin" ],
"localadmin_id": [ "8" ],
"localadmin_a": [ null ],
"neighbourhood": [ "override neighbourhood" ],
"neighbourhood_id": [ "9" ],
"neighbourhood_a": [ null ]
"country": [
"override country"
],
"country_id": [
"1"
],
"country_a": [
null
],
"macroregion": [
"override macroregion"
],
"macroregion_id": [
"2"
],
"macroregion_a": [
null
],
"region": [
"override region"
],
"region_id": [
"3"
],
"region_a": [
null
],
"macrocounty": [
"override macrocounty"
],
"macrocounty_id": [
"4"
],
"macrocounty_a": [
null
],
"county": [
"override county"
],
"county_id": [
"5"
],
"county_a": [
null
],
"borough": [
"override borough"
],
"borough_id": [
"6"
],
"borough_a": [
null
],
"locality": [
"override locality"
],
"locality_id": [
"7"
],
"locality_a": [
null
],
"localadmin": [
"override localadmin"
],
"localadmin_id": [
"8"
],
"localadmin_a": [
null
],
"neighbourhood": [
"override neighbourhood"
],
"neighbourhood_id": [
"9"
],
"neighbourhood_a": [
null
]
},
"address_parts": {
"unit": "1",
"number": "100",
"street": "Main St",
"zip": "input postcode"
Expand All @@ -50,21 +130,48 @@
},
"source": "openaddresses",
"layer": "address",
"source_id": "data/input_file_1:0"
"source_id": "data/input_file_1:1"
}
},
{
"_index": "pelias",
"_type": "address",
"_id": "data/input_file_1:1",
"_id": "data/input_file_1:2",
"data": {
"name": {
"default": "100 2 Main St"
},
"phrase": {
"default": "100 2 Main St"
},
"address_parts": {
"unit": "2",
"number": "100",
"street": "Main St",
"zip": "input postcode"
},
"center_point": {
"lon": 21.313131,
"lat": 12.131313
},
"source": "openaddresses",
"layer": "address",
"source_id": "data/input_file_1:2"
}
},
{
"_index": "pelias",
"_type": "address",
"_id": "data/input_file_1:3",
"data": {
"name": {
"default": "200 Main St"
"default": "200 tv Main St"
},
"phrase": {
"default": "200 Main St"
"default": "200 tv Main St"
},
"address_parts": {
"unit": "tv",
"number": "200",
"street": "Main St"
},
Expand All @@ -74,13 +181,13 @@
},
"source": "openaddresses",
"layer": "address",
"source_id": "data/input_file_1:1"
"source_id": "data/input_file_1:3"
}
},
{
"_index": "pelias",
"_type": "address",
"_id": "data/input_file_1:4",
"_id": "data/input_file_1:6",
"data": {
"name": {
"default": "300 Main St"
Expand All @@ -98,7 +205,7 @@
},
"source": "openaddresses",
"layer": "address",
"source_id": "data/input_file_1:4"
"source_id": "data/input_file_1:6"
}
},
{
Expand Down Expand Up @@ -149,4 +256,4 @@
"source_id": "data/input_file_2:2"
}
}
]
]
8 changes: 5 additions & 3 deletions test/data/input_file_1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
LON,LAT,NUMBER,STREET,UNIT,CITY,DISTRICT,REGION,POSTCODE,ID
21.212121,12.121212,100,Main St,,input city,input district,input region,input postcode,GOOD RECORD
31.313131,13.131313, 200 , Main St ,,,,,,GOOD RECORD WITH FIELD TRIMMING
41.414141,14.141414,0,Main St,,,,,,WILL BE IGNORED SINCE NUMBER=0
21.111111,12.111111,100,Main St,,input city,input district,input region,input postcode,GOOD RECORD
21.212121,12.121212,100,Main St,1,input city,input district,input region,input postcode,GOOD RECORD
21.313131,12.131313,100,Main St,2,input city,input district,input region,input postcode,GOOD RECORD
31.313131,13.131313, 200 , Main St , tv ,,,,,GOOD RECORD WITH FIELD TRIMMING
41.414141,14.141414,0,Main St,1 tv,,,,,WILL BE IGNORED SINCE NUMBER=0
51.515151,15.151515,00,Elm St,,,,,,WILL BE IGNORED SINCE NUMBER IS REDUCEABLE TO 0
61.616161,16.161616,00300,Main St,,,,,,REMOVES LEADING ZEROES
20 changes: 20 additions & 0 deletions test/streams/documentStream.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,26 @@ tape( 'documentStream catches records with no street', function(test) {
});
});

tape( 'documentStream does not set unit if unit is emptystring', function(test) {
const input = {
NUMBER: '5',
STREET: '101st Avenue',
LAT: 5,
LON: 6,
POSTCODE: '12345',
UNIT: ''
};
const stats = { badRecordCount: 0 };
const documentStream = DocumentStream.create('prefix', stats);

test_stream([input], documentStream, function(err, actual) {
test.equal(actual.length, 1, 'the document should be pushed' );
test.equal(stats.badRecordCount, 0, 'bad record count unchanged');
test.equal(actual[0].getAddress('unit', undefined));
test.end();
});
});

tape( 'documentStream does not set zipcode if zipcode is emptystring', function(test) {
const input = {
NUMBER: '5',
Expand Down