Skip to content

Commit

Permalink
Add ISO 3166-1 code and improuved territories loading
Browse files Browse the repository at this point in the history
  • Loading branch information
tmerlier committed Feb 6, 2017
1 parent 955fe60 commit 4e4c800
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 53 deletions.
2 changes: 1 addition & 1 deletion lib/countryHelpers.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const { initFields } = require('./helpers');

const initCountryFields = initFields({
default: ['nom', 'code', 'iso2', 'iso3', 'territories'],
default: ['nom', 'code', 'iso2', 'iso3', 'num', 'territories'],
base: ['nom', 'code'],
});

Expand Down
46 changes: 29 additions & 17 deletions lib/integration/countries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,35 +11,47 @@ function init(ctx, next) {

ctx.getCountry = (code) => {
if (!ctx.countries.has(code)) {
const country = { code };
const country = { code, territories: new Set() };
ctx.countries.set(code, country);
}
return ctx.countries.get(code);
};

ctx.getTerritories = (country, srcPath, callBack) => {
country.territories = [];
next();
}

/* Chargement des territoires */
function loadTerritories(options = {}) {
return function (ctx, next) {
ctx.debug('Chargement du jeu de données pays2016 ');
let count = 0;
const srcPath = options.srcPath || __dirname + '/../../forced_data/pays2016.tsv';

fs.createReadStream(srcPath)
.pipe(iconv.decodeStream('win1252'))
.on('error', next)
.pipe(parse({ delimiter: '\t', columns: true }))
.on('error', next)
.pipe(t((data, enc, cb) => {
if (data.COG === country.code && data.ACTUAL == 3) {
const territory = { nom: data.LIBCOG };
country.territories.push(territory);
const code = data.COG;
if (data.ACTUAL === '3') {
const territory = {
nom: data.LIBCOG,
iso2: data.CODEISO2,
iso3: data.CODEISO2,
num: data.CODENUM3,
};
ctx.getCountry(code).territories.add(territory);
count++;
}
cb();
}))
.on('error', next)
.on('finish', () => {
// if (territories.length) console.log(`Ajout de ${territories.length} territore pour ${code} !`);
callBack();
ctx.debug('Nombre de territoires chargées : %d', count);
next();
});
};

next();
}

/* Chargement des pays */
Expand All @@ -56,16 +68,15 @@ function loadCountries(options = {}) {
.on('error', next)
.pipe(t((data, enc, cb) => {
const code = data.COG;
if (code !== 'XXXXX' && data.ACTUAL !== 3) {
if (code !== 'XXXXX' && data.ACTUAL !== '3') {
const country = ctx.getCountry(code);
country.nom = data.LIBCOG;
country.iso2 = data.CODEISO2;
country.iso3 = data.CODEISO3;
country.num = data.CODENUM3;
count++;
ctx.getTerritories(country, srcPath, cb);
} else {
cb();
}
cb();
}))
.on('error', next)
.on('finish', () => {
Expand All @@ -86,9 +97,10 @@ function serialize(options = {}) {
} else {
streamify(Array.from(ctx.countries.values()))
.on('error', next)
.pipe(t((dep, enc, cb) => {
.pipe(t((country, enc, cb) => {
country.territories = Array.from(country.territories).sort();
count++;
cb(null, dep);
cb(null, country);
}))
.on('error', next)
.pipe(JSONStream.stringify())
Expand All @@ -104,4 +116,4 @@ function serialize(options = {}) {
}

/* Exports */
module.exports = { init, serialize, loadCountries };
module.exports = { init, serialize, loadCountries, loadTerritories };
1 change: 1 addition & 0 deletions lib/integration/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ function integrate(done) {
// Pays
countries.init,
countries.loadCountries(),
countries.loadTerritories(),
countries.serialize(),
], done);
}
Expand Down
2 changes: 1 addition & 1 deletion test/countryHelpers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ describe('countryHelpers', function () {
it('empty request should return default fields', function (done) {
runTestCase(
{},
['nom', 'code', 'iso2', 'iso3', 'territories'],
['nom', 'code', 'iso2', 'iso3', 'num', 'territories'],
done
);
});
Expand Down
94 changes: 61 additions & 33 deletions test/countryIntegration.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* eslint-env mocha */
const { init, serialize, loadCountries } = require('../lib/integration/countries');
const { init, serialize, loadCountries, loadTerritories } = require('../lib/integration/countries');
const expect = require('expect.js');

describe('#integration countries', () => {
Expand All @@ -16,10 +16,6 @@ describe('#integration countries', () => {
expect(ctx.countries).to.be.a(Map);
expect(ctx.countries.size).to.be(0);
});
it('should set ctx.getCountry function',
() => expect(ctx.getCountry).to.be.a(Function));
it('should set ctx.getTerritories function',
() => expect(ctx.getTerritories).to.be.a(Function));
});

describe('getCountry()', () => {
Expand All @@ -30,15 +26,17 @@ describe('#integration countries', () => {
it('should return a country with given code', () => {
const country = ctx.getCountry('42');
expect(country).to.be.an(Object);
expect(country).to.only.have.keys('code');
expect(country).to.have.keys('code');
expect(country).to.have.keys('territories');
expect(country.code).to.be('42');
});
it('should store the country', () => {
ctx.getCountry('21');
expect(ctx.countries.size).to.be(1);
expect(ctx.countries.has('21')).to.be.ok();
const country = ctx.countries.get('21');
expect(country).to.only.have.keys('code');
expect(country).to.have.keys('code');
expect(country).to.have.keys('territories');
expect(country.code).to.be('21');
});
});
Expand All @@ -51,7 +49,8 @@ describe('#integration countries', () => {
it('should return a country with given code', () => {
const country = ctx.getCountry('11');
expect(country).to.be.an(Object);
expect(country).to.only.have.keys('code');
expect(country).to.have.keys('code');
expect(country).to.have.keys('territories');
expect(country.code).to.be('11');
});
it('should have no impact on storage', () => {
Expand All @@ -61,29 +60,6 @@ describe('#integration countries', () => {
});
});
});

describe('getTerritories()', () => {
describe('Country has no territory', () => {
it('should assign an empty array to country.territories', () => {
const country = ctx.getCountry('42');
const srcPath = __dirname + '/integration-data/countries.tsv';
ctx.getTerritories(country, srcPath, () => expect(country.territories).to.eql([]));
});
});

describe('Country has several territories', () => {
it('should assign the 3 territories that have the same code as the country.', () => {
const country = ctx.getCountry('123456');
const srcPath = __dirname + '/integration-data/countries-territories.tsv';
ctx.getTerritories(country, srcPath, () => true);
expect(country.territories).to.eql([
{ nom: 'TEST1' },
{ nom: 'TEST2' },
{ nom: 'TEST3' },
]);
});
});
});
});

describe('loadCountries()', () => {
Expand All @@ -97,7 +73,7 @@ describe('#integration countries', () => {
country.code = code;
return country;
},
getTerritories: (country, srcPath, callBack) => {
loadTerritories: (country, srcPath, callBack) => {
country.territories = [];
callBack() ;
},
Expand All @@ -113,14 +89,64 @@ describe('#integration countries', () => {
nom: 'TEST',
iso2: 'TS',
iso3: 'TST',
territories: [],
num: '250',
});
done();
});
});
});
});

describe('loadTerritories()', () => {
describe('Processing a file containing a relation with known COG code', () => {
it('should associate territories to the country', done => {
const country = { code: '123456', territories: new Set() };
const ctx = {
countries: { has: () => true },
debug: () => {},
getCountry: () => country,
};

loadTerritories({ srcPath: __dirname + '/integration-data/countries-territories.tsv' })(ctx, err => {
expect(err).to.be(undefined);
expect(Array.from(country.territories)).to.eql(
[
{
'iso2': 'T1',
'iso3': 'T1',
'nom': 'TEST1',
'num': '251',
},
{
'iso2': 'T2',
'iso3': 'T2',
'nom': 'TEST2',
'num': '252',
},
]);
done();
});
});
});

describe('Country has no territories', () => {
it('should not associate any territory', done => {
const country = { code: '654321', territories: new Set() };
const ctx = {
countries: { has: () => true },
debug: () => {},
getCountry: (code) => code === country.code ? country : { territories: new Set() },
};

loadTerritories({ srcPath: __dirname + '/integration-data/countries-territories.tsv' })(ctx, err => {
expect(err).to.be(undefined);
expect(Array.from(country.territories)).to.eql([]);
done();
});
});
});
});

describe('serialize()', () => {
describe('No country', () => {
it('should throw an error', done => {
Expand All @@ -140,6 +166,7 @@ describe('#integration countries', () => {
['42', {
code: '42',
nom: 'Test',
territories: new Set(),
}],
]) };
serialize({ destPath: __dirname + '/' + path })(ctx, err => {
Expand All @@ -149,6 +176,7 @@ describe('#integration countries', () => {
expect(countries[0]).to.eql({
code: '42',
nom: 'Test',
territories: [],
});
done();
});
Expand Down
1 change: 0 additions & 1 deletion test/integration-data/countries-territories.tsv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,5 @@ COG ACTUAL CAPAY CRPAY ANI LIBCOG LIBENR ANCNOM CODEISO2 CODEISO3 CODENUM3
123456 1 TEST TEST TS TST 250
123456 3 TEST1 TEST1 T1 TS1 251
123456 3 TEST2 TEST2 T2 TS2 252
123456 3 TEST3 TEST3 T3 TS3 253
654321 2 OTHER OTHER OT OTH 254
615243 1 TRAP TRAP TP TRP 255

0 comments on commit 4e4c800

Please sign in to comment.