diff --git a/README.md b/README.md index be9d2fe..52504da 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https: - [Validate](#validate) - [Deduplicate](#deduplicate) - [InvertAllow](#invertallow) + - [RemoveEmptyLines](#removeemptylines) + - [TrimLines](#trimlines) + - [InsertFinalNewLine](#insertfinalnewline) - [How to build](#how-to-build) ## Usage @@ -131,9 +134,9 @@ Examples: ### API -``` -npm i @adguard/hostlist-compiler -``` +Install: `npm i @adguard/hostlist-compiler` or `yarn add @adguard/hostlist-compiler` + +#### JavaScript example: ```javascript const compile = require("@adguard/hostlist-compiler"); @@ -155,6 +158,28 @@ async function main() { main(); ``` +#### TypeScript example: + +```typescript +import HostlistCompiler, { IConfiguration as HostlistCompilerConfiguration } from '@adguard/hostlist-compiler'; +// or: +// import compiler, { IConfiguration as CompilerConfiguration } from '@adguard/hostlist-compiler'; +import { writeFileSync } from 'fs'; + +(async () => { + // Configuration + const config: HostlistCompilerConfiguration = { + // ... + }; + + // Compile filters + const result = await HostlistCompiler(config); + + // Write to file + writeFileSync('custom-adguard-dns.txt', result.join('\n')); +})(); +``` + ## Transformations Here is the full list of transformations that are available: @@ -165,6 +190,9 @@ Here is the full list of transformations that are available: 4. `Validate` 5. `Deduplicate` 6. `InvertAllow` +7. `RemoveEmptyLines` +8. `TrimLines` +9. `InsertFinalNewLine` Please note that these transformations are are always applied in the order specified here. @@ -266,6 +294,80 @@ Here's what we will have after applying this transformation: @@rule2 ``` +### RemoveEmptyLines + +This is a very simple transformation that removes empty lines. + +**Example:** + +Original list: + +``` +rule1 + +rule2 + + +rule3 +``` + +Here's what we will have after applying this transformation: + +``` +rule1 +rule2 +rule3 +``` + +### TrimLines + +This is a very simple transformation that removes leading and trailing spaces/tabs. + +**Example:** + +Original list: + +``` +rule1 + rule2 +rule3 + rule4 +``` + +Here's what we will have after applying this transformation: + +``` +rule1 +rule2 +rule3 +rule4 +``` + +### InsertFinalNewLine + +This is a very simple transformation that inserts a final new line. + +**Example:** + +Original list: + +``` +rule1 +rule2 +rule3 +``` + +Here's what we will have after applying this transformation: + +``` +rule1 +rule2 +rule3 + +``` + +`RemoveEmptyLines` transformation has no effect on this new line because it precedes this transformation in the execution queue. + ## How to build - `yarn install` - installs dependencies diff --git a/package.json b/package.json index c1bd44e..07e89e4 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "1.0.13", "description": "A simple tool that compiles hosts blocklists from multiple sources", "main": "src/index.js", + "types": "src/index.d.ts", "repository": "https://github.com/AdguardTeam/HostlistCompiler", "author": "AdGuard", "license": "GPL-3.0", diff --git a/src/compile-source.js b/src/compile-source.js index 7ba9ff8..4ddb67b 100644 --- a/src/compile-source.js +++ b/src/compile-source.js @@ -17,7 +17,7 @@ const { transform } = require('./transformations/transform'); * Compiles an individual source according to it's configuration. * * @param {ListSource} source - source configuration. - * @returns {Array} array with the source rules + * @returns {Promise>} array with the source rules */ async function compileSource(source) { consola.info(`Start compiling ${source.source}`); diff --git a/src/filter.js b/src/filter.js index d3cebab..807636e 100644 --- a/src/filter.js +++ b/src/filter.js @@ -7,7 +7,7 @@ const ruleUtils = require('./rule'); * and return a final array with all the lines from all files. * * @param {Array} sources - array of URLs. - * @returns {Array} array with all non-empty and non-comment lines. + * @returns {Promise>} array with all non-empty and non-comment lines. */ async function downloadAll(sources) { let list = []; @@ -32,7 +32,7 @@ async function downloadAll(sources) { * @param {Array} rules - array of rules to apply * @param {Array} sources - array of rules sources * (can be local or remote files) - * @returns {Array} a list of wildcards to apply + * @returns {Promise>} a list of wildcards to apply */ async function prepareWildcards(rules, sources) { let list = []; diff --git a/src/index.d.ts b/src/index.d.ts new file mode 100644 index 0000000..4669375 --- /dev/null +++ b/src/index.d.ts @@ -0,0 +1,59 @@ +declare module '@adguard/hostlist-compiler' { + export type SourceType = 'adblock' | 'hosts'; + export type Transformation = 'RemoveComments' | 'Compress' | 'RemoveModifiers' | 'Validate' | 'Deduplicate' | 'InvertAllow' | 'RemoveEmptyLines' | 'TrimLines' | 'InsertFinalNewLine'; + + /** A source for the filter list */ + export interface ISource { + /** Name of the source */ + name?: string; + /** Path to a file or a URL */ + source: string; + /** Type of the source */ + type?: SourceType; + /** A list of the transformations that will be applied */ + transformations?: Transformation[]; + /** A list of rules (or wildcards) to exclude from the source. */ + exclusions?: string[]; + /** An array of exclusions sources. */ + exclusions_sources?: string[]; + /** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */ + inclusions?: string[]; + /** A list of files with inclusions. */ + inclusions_sources?: string[]; + } + + /** Configuration for the hostlist compiler */ + export interface IConfiguration { + /** Filter list name */ + name: string; + /** Filter list description */ + description?: string; + /** Filter list homepage */ + homepage?: string; + /** Filter list license */ + license?: string; + /** An array of the filter list sources */ + sources: ISource[]; + /** A list of the transformations that will be applied */ + transformations?: Transformation[]; + /** A list of rules (or wildcards) to exclude from the source. */ + exclusions?: string[]; + /** An array of exclusions sources. */ + exclusions_sources?: string[]; + /** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */ + inclusions?: string[]; + /** A list of files with inclusions. */ + inclusions_sources?: string[]; + } + + /** + * Compiles a filter list using the specified configuration. + * + * @param {*} configuration - compilation configuration. + See the repo README for the details on it. + * @returns {Promise>} the array of rules. + */ + declare async function compile(configuration: IConfiguration): Promise; + + export default compile; +} diff --git a/src/index.js b/src/index.js index 63f6d10..9615c34 100644 --- a/src/index.js +++ b/src/index.js @@ -65,7 +65,7 @@ function prepareSourceHeader(source) { * * @param {*} configuration - compilation configuration. See the repo README for the details on it. - * @returns {Array} the array of rules. + * @returns {Promise>} the array of rules. */ async function compile(configuration) { consola.info('Starting the compiler'); diff --git a/src/schemas/configuration.schema.json b/src/schemas/configuration.schema.json index bda3486..cb3d559 100644 --- a/src/schemas/configuration.schema.json +++ b/src/schemas/configuration.schema.json @@ -41,7 +41,10 @@ "Compress", "Validate", "Deduplicate", - "InvertAllow" + "InvertAllow", + "RemoveEmptyLines", + "TrimLines", + "InsertFinalNewLine" ] } }, @@ -113,7 +116,10 @@ "Compress", "Validate", "Deduplicate", - "InvertAllow" + "InvertAllow", + "RemoveEmptyLines", + "TrimLines", + "InsertFinalNewLine" ] } }, diff --git a/src/transformations/exclude.js b/src/transformations/exclude.js index 6ef3826..da6ea8d 100644 --- a/src/transformations/exclude.js +++ b/src/transformations/exclude.js @@ -9,7 +9,7 @@ const filterUtils = require('../filter'); * @param {Array} exclusions - array of exclusions to apply * @param {Array} exclusionsSources - array of exclusion sources * (can be a local or remote file) - * @returns {Array} filtered array of rules + * @returns {Promise>} filtered array of rules */ async function exclude(rules, exclusions, exclusionsSources) { if (_.isEmpty(exclusions) && _.isEmpty(exclusionsSources)) { diff --git a/src/transformations/include.js b/src/transformations/include.js index 9f2eefd..dd53859 100644 --- a/src/transformations/include.js +++ b/src/transformations/include.js @@ -9,7 +9,7 @@ const filterUtils = require('../filter'); * @param {Array} inclusions - array of inclusions to apply * @param {Array} inclusionsSources - array of inclusions' sources * (can be a local or remote file) - * @returns {Array} filtered array of rules + * @returns {Promise>} filtered array of rules */ async function include(rules, inclusions, inclusionsSources) { if (_.isEmpty(inclusions) && _.isEmpty(inclusionsSources)) { diff --git a/src/transformations/insert-final-new-line.js b/src/transformations/insert-final-new-line.js new file mode 100644 index 0000000..7cd40ae --- /dev/null +++ b/src/transformations/insert-final-new-line.js @@ -0,0 +1,17 @@ +const consola = require('consola'); + +/** + * This is a very simple transformation that inserts a final new line. + * + * @param {Array} lines - lines/rules to transform + * @returns {Array} filtered lines/rules + */ +function insertFinalNewLine(lines) { + if (lines.length === 0 || (lines.length > 0 && lines[lines.length - 1].trim() !== '')) { + lines.push(''); + } + consola.info('Final new line inserted'); + return lines; +} + +module.exports = insertFinalNewLine; diff --git a/src/transformations/remove-empty-lines.js b/src/transformations/remove-empty-lines.js new file mode 100644 index 0000000..379a3fe --- /dev/null +++ b/src/transformations/remove-empty-lines.js @@ -0,0 +1,15 @@ +const consola = require('consola'); + +/** + * This is a very simple transformation that removes empty lines. + * + * @param {Array} lines - lines/rules to transform + * @returns {Array} filtered lines/rules + */ +function removeEmptyLines(lines) { + const filtered = lines.filter((line) => line.trim().length); + consola.info(`Removed ${lines.length - filtered.length} empty lines`); + return filtered; +} + +module.exports = removeEmptyLines; diff --git a/src/transformations/transform.js b/src/transformations/transform.js index 9c76f3a..559fa5b 100644 --- a/src/transformations/transform.js +++ b/src/transformations/transform.js @@ -6,6 +6,9 @@ const include = require('./include'); const deduplicate = require('./deduplicate'); const compress = require('./compress'); const invertAllow = require('./invertallow'); +const removeEmptyLines = require('./remove-empty-lines'); +const trimLines = require('./trim-lines'); +const insertFinalNewLine = require('./insert-final-new-line'); /** * Enum with all available transformations @@ -17,6 +20,9 @@ const TRANSFORMATIONS = Object.freeze({ Validate: 'Validate', Deduplicate: 'Deduplicate', InvertAllow: 'InvertAllow', + RemoveEmptyLines: 'RemoveEmptyLines', + TrimLines: 'TrimLines', + InsertFinalNewLine: 'InsertFinalNewLine', }); /** @@ -25,7 +31,7 @@ const TRANSFORMATIONS = Object.freeze({ * @param {Array} rules - rules to transform * @param {*} configuration - transformation configuration. * @param {Array} transformations - a list of transformations to apply to the rules. - * @returns {Array} rules after applying all transformations. + * @returns {Promise>} rules after applying all transformations. */ async function transform(rules, configuration, transformations) { // If none specified -- apply all transformationss @@ -59,6 +65,15 @@ async function transform(rules, configuration, transformations) { if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) { transformed = deduplicate(transformed); } + if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) { + transformed = removeEmptyLines(transformed); + } + if (transformations.indexOf(TRANSFORMATIONS.TrimLines) !== -1) { + transformed = trimLines(transformed); + } + if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) { + transformed = insertFinalNewLine(transformed); + } return transformed; } diff --git a/src/transformations/trim-lines.js b/src/transformations/trim-lines.js new file mode 100644 index 0000000..8371f13 --- /dev/null +++ b/src/transformations/trim-lines.js @@ -0,0 +1,16 @@ +const consola = require('consola'); +const _ = require('lodash'); + +/** + * This is a very simple transformation that removes leading and trailing spaces/tabs. + * + * @param {Array} lines - lines/rules to transform + * @returns {Array} filtered lines/rules + */ +function trimLines(lines) { + const transformed = lines.map((line) => _.trim(line, [' ', '\t'])); + consola.info('Lines trimmed.'); + return transformed; +} + +module.exports = trimLines; diff --git a/src/utils.js b/src/utils.js index 64e7b2d..1486fec 100644 --- a/src/utils.js +++ b/src/utils.js @@ -16,7 +16,7 @@ function isURL(str) { * Downloads (or reads from the disk) the specified source * * @param {*} urlOrPath url or path to a file - * @returns {String} contents of the files + * @returns {Promise} contents of the files */ async function download(urlOrPath) { let str = ''; diff --git a/test/transformations/insert-final-new-line.test.js b/test/transformations/insert-final-new-line.test.js new file mode 100644 index 0000000..b98b7fd --- /dev/null +++ b/test/transformations/insert-final-new-line.test.js @@ -0,0 +1,48 @@ +const insertFinalNewLine = require('../../src/transformations/insert-final-new-line'); + +describe('Insert final new line', () => { + it('test with empty file', () => { + const rules = []; + const filtered = insertFinalNewLine(rules); + expect(filtered).toEqual(['']); + }); + it('test with one empty line', () => { + const rules = ['']; + const filtered = insertFinalNewLine(rules); + expect(filtered).toEqual(['']); + }); + it('test with one rule and one empty lines', () => { + const rules = [ + 'rule1', + '', + ]; + const filtered = insertFinalNewLine(rules); + expect(filtered).toEqual([ + 'rule1', + '', + ]); + }); + it('test with many empty lines', () => { + const rules = [ + 'rule1', + '', + '', + ]; + const filtered = insertFinalNewLine(rules); + expect(filtered).toEqual([ + 'rule1', + '', + '', + ]); + }); + it('test with one rule', () => { + const rules = [ + 'rule1', + ]; + const filtered = insertFinalNewLine(rules); + expect(filtered).toEqual([ + 'rule1', + '', + ]); + }); +}); diff --git a/test/transformations/remove-empty-lines.test.js b/test/transformations/remove-empty-lines.test.js new file mode 100644 index 0000000..86df05f --- /dev/null +++ b/test/transformations/remove-empty-lines.test.js @@ -0,0 +1,77 @@ +const removeEmptyLines = require('../../src/transformations/remove-empty-lines'); + +describe('Remove empty lines', () => { + it('test with no empty lines', () => { + const rules = [ + 'rule1', + 'rule2', + ]; + const filtered = removeEmptyLines(rules); + expect(filtered).toEqual([ + 'rule1', + 'rule2', + ]); + }); + it('test with one empty line at the end', () => { + const rules = [ + 'rule1', + 'rule2', + '', + ]; + const filtered = removeEmptyLines(rules); + expect(filtered).toEqual([ + 'rule1', + 'rule2', + ]); + }); + it('test with one empty line between the rules', () => { + const rules = [ + 'rule1', + '', + 'rule2', + ]; + const filtered = removeEmptyLines(rules); + expect(filtered).toEqual([ + 'rule1', + 'rule2', + ]); + }); + it('test with two empty line', () => { + const rules = [ + 'rule1', + '', + '', + 'rule2', + ]; + const filtered = removeEmptyLines(rules); + expect(filtered).toEqual([ + 'rule1', + 'rule2', + ]); + }); + it('test with comments, empty lines', () => { + const rules = [ + '! aaa', + 'rule1', + '', + '! bbb', + '', + '', + 'rule2', + '', + '', + '', + '!ccc', + '', + '', + ]; + const filtered = removeEmptyLines(rules); + expect(filtered).toEqual([ + '! aaa', + 'rule1', + '! bbb', + 'rule2', + '!ccc', + ]); + }); +}); diff --git a/test/transformations/trim-lines.test.js b/test/transformations/trim-lines.test.js new file mode 100644 index 0000000..0352247 --- /dev/null +++ b/test/transformations/trim-lines.test.js @@ -0,0 +1,52 @@ +const trimLines = require('../../src/transformations/trim-lines'); + +describe('Trim lines', () => { + it('test with empty file', () => { + const rules = []; + const filtered = trimLines(rules); + expect(filtered).toEqual([]); + }); + it('test with one empty line', () => { + const rules = ['']; + const filtered = trimLines(rules); + expect(filtered).toEqual(['']); + }); + it('test with three rules', () => { + const rules = [ + 'rule1', + ' rule2 ', + '', + ' rule3 ', + ]; + const filtered = trimLines(rules); + expect(filtered).toEqual([ + 'rule1', + 'rule2', + '', + 'rule3', + ]); + }); + it('test with three rules and comments', () => { + const rules = [ + 'rule1', + ' rule2 ', + '', + '', + '! comment ', + ' rule3 ', + ' ! comment multiple words ', + '', + ]; + const filtered = trimLines(rules); + expect(filtered).toEqual([ + 'rule1', + 'rule2', + '', + '', + '! comment', + 'rule3', + '! comment multiple words', + '', + ]); + }); +});