TS declarations / RemoveEmptyLines / InsertFinalNewLine / TrimLines

README.md

@@ -15,6 +15,9 @@ This is a simple tool that makes it easier to compile a [hosts blocklist](https:
   - [Validate](#validate)
   - [Deduplicate](#deduplicate)
   - [InvertAllow](#invertallow)
+  - [RemoveEmptyLines](#removeemptylines)
+  - [TrimLines](#trimlines)
+  - [InsertFinalNewLine](#insertfinalnewline)
 - [How to build](#how-to-build)
 
 ## <a id="usage"></a> Usage
@@ -131,9 +134,9 @@ Examples:
 
 ### <a id="api"></a> API
 
-```
-npm i @adguard/hostlist-compiler
-```
+Install: `npm i @adguard/hostlist-compiler` or `yarn add @adguard/hostlist-compiler`
+
+#### JavaScript example:
 
 ```javascript
 const compile = require("@adguard/hostlist-compiler");
@@ -155,6 +158,28 @@ async function main() {
 main();
 ```
 
+#### TypeScript example:
+
+```typescript
+import HostlistCompiler, { IConfiguration as HostlistCompilerConfiguration } from '@adguard/hostlist-compiler';
+// or:
+// import compiler, { IConfiguration as CompilerConfiguration } from '@adguard/hostlist-compiler';
+import { writeFileSync } from 'fs';
+
+(async () => {
+    // Configuration
+    const config: HostlistCompilerConfiguration = {
+        // ...
+    };
+
+    // Compile filters
+    const result = await HostlistCompiler(config);
+
+    // Write to file
+    writeFileSync('custom-adguard-dns.txt', result.join('\n'));
+})();
+```
+
 ## <a id="transformations"></a> Transformations
 
 Here is the full list of transformations that are available:
@@ -165,6 +190,9 @@ Here is the full list of transformations that are available:
 4. `Validate`
 5. `Deduplicate`
 6. `InvertAllow`
+7. `RemoveEmptyLines`
+8. `TrimLines`
+9. `InsertFinalNewLine`
 
 Please note that these transformations are are always applied in the order specified here.
 
@@ -266,6 +294,80 @@ Here's what we will have after applying this transformation:
 @@rule2
 ```
 
+### <a id="removeemptylines"></a> RemoveEmptyLines
+
+This is a very simple transformation that removes empty lines.
+
+**Example:**
+
+Original list:
+
+```
+rule1
+
+rule2
+
+
+rule3
+```
+
+Here's what we will have after applying this transformation:
+
+```
+rule1
+rule2
+rule3
+```
+
+### <a id="trimlines"></a> TrimLines
+
+This is a very simple transformation that removes leading and trailing spaces/tabs.
+
+**Example:**
+
+Original list:
+
+```
+rule1
+   rule2
+rule3
+		rule4
+```
+
+Here's what we will have after applying this transformation:
+
+```
+rule1
+rule2
+rule3
+rule4
+```
+
+### <a id="insertfinalnewline"></a> InsertFinalNewLine
+
+This is a very simple transformation that inserts a final new line.
+
+**Example:**
+
+Original list:
+
+```
+rule1
+rule2
+rule3
+```
+
+Here's what we will have after applying this transformation:
+
+```
+rule1
+rule2
+rule3
+
+```
+
+`RemoveEmptyLines` transformation has no effect on this new line because it precedes this transformation in the execution queue.
+
 ## <a id="how-to-build"></a> How to build
 
 - `yarn install` - installs dependencies

package.json

@@ -3,6 +3,7 @@
   "version": "1.0.13",
   "description": "A simple tool that compiles hosts blocklists from multiple sources",
   "main": "src/index.js",
+  "types": "src/index.d.ts",
   "repository": "https://github.com/AdguardTeam/HostlistCompiler",
   "author": "AdGuard",
   "license": "GPL-3.0",

src/compile-source.js

@@ -17,7 +17,7 @@ const { transform } = require('./transformations/transform');
  * Compiles an individual source according to it's configuration.
  *
  * @param {ListSource} source - source configuration.
- * @returns {Array<string>} array with the source rules
+ * @returns {Promise<Array<string>>} array with the source rules
  */
 async function compileSource(source) {
     consola.info(`Start compiling ${source.source}`);

src/filter.js

@@ -7,7 +7,7 @@ const ruleUtils = require('./rule');
  * and return a final array with all the lines from all files.
  *
  * @param {Array<String>} sources - array of URLs.
- * @returns {Array<String>} array with all non-empty and non-comment lines.
+ * @returns {Promise<Array<String>>} array with all non-empty and non-comment lines.
  */
 async function downloadAll(sources) {
     let list = [];
@@ -32,7 +32,7 @@ async function downloadAll(sources) {
  * @param {Array<String>} rules - array of rules to apply
  * @param {Array<String>} sources - array of rules sources
  * (can be local or remote files)
- * @returns {Array<utils.Wildcard>} a list of wildcards to apply
+ * @returns {Promise<Array<utils.Wildcard>>} a list of wildcards to apply
  */
 async function prepareWildcards(rules, sources) {
     let list = [];

src/index.d.ts

@@ -0,0 +1,59 @@
+declare module '@adguard/hostlist-compiler' {
+    export type SourceType = 'adblock' | 'hosts';
+    export type Transformation = 'RemoveComments' | 'Compress' | 'RemoveModifiers' | 'Validate' | 'Deduplicate' | 'InvertAllow' | 'RemoveEmptyLines' | 'TrimLines' | 'InsertFinalNewLine';
+
+    /** A source for the filter list */
+    export interface ISource {
+        /** Name of the source */
+        name?: string;
+        /** Path to a file or a URL */
+        source: string;
+        /** Type of the source */
+        type?: SourceType;
+        /** A list of the transformations that will be applied */
+        transformations?: Transformation[];
+        /** A list of rules (or wildcards) to exclude from the source. */
+        exclusions?: string[];
+        /** An array of exclusions sources. */
+        exclusions_sources?: string[];
+        /** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
+        inclusions?: string[];
+        /** A list of files with inclusions. */
+        inclusions_sources?: string[];
+    }
+
+    /** Configuration for the hostlist compiler */
+    export interface IConfiguration {
+        /** Filter list name */
+        name: string;
+        /** Filter list description */
+        description?: string;
+        /** Filter list homepage */
+        homepage?: string;
+        /** Filter list license */
+        license?: string;
+        /** An array of the filter list sources */
+        sources: ISource[];
+        /** A list of the transformations that will be applied */
+        transformations?: Transformation[];
+        /** A list of rules (or wildcards) to exclude from the source. */
+        exclusions?: string[];
+        /** An array of exclusions sources. */
+        exclusions_sources?: string[];
+        /** A list of wildcards to include from the source. All rules that don't match these wildcards won't be included. */
+        inclusions?: string[];
+        /** A list of files with inclusions. */
+        inclusions_sources?: string[];
+    }
+
+    /**
+     * Compiles a filter list using the specified configuration.
+     *
+     * @param {*} configuration - compilation configuration.
+    See the repo README for the details on it.
+     * @returns {Promise<Array<string>>} the array of rules.
+     */
+    declare async function compile(configuration: IConfiguration): Promise<string[]>;
+
+    export default compile;
+}

src/index.js

@@ -65,7 +65,7 @@ function prepareSourceHeader(source) {
  *
  * @param {*} configuration - compilation configuration.
 See the repo README for the details on it.
- * @returns {Array<string>} the array of rules.
+ * @returns {Promise<Array<string>>} the array of rules.
  */
 async function compile(configuration) {
     consola.info('Starting the compiler');

src/schemas/configuration.schema.json

@@ -41,7 +41,10 @@
           "Compress",
           "Validate",
           "Deduplicate",
-          "InvertAllow"
+          "InvertAllow",
+          "RemoveEmptyLines",
+          "TrimLines",
+          "InsertFinalNewLine"
         ]
       }
     },
@@ -113,7 +116,10 @@
               "Compress",
               "Validate",
               "Deduplicate",
-              "InvertAllow"
+              "InvertAllow",
+              "RemoveEmptyLines",
+              "TrimLines",
+              "InsertFinalNewLine"
             ]
           }
         },

src/transformations/exclude.js

@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
  * @param {Array<String>} exclusions - array of exclusions to apply
  * @param {Array<String>} exclusionsSources - array of exclusion sources
  * (can be a local or remote file)
- * @returns {Array<String>} filtered array of rules
+ * @returns {Promise<Array<String>>} filtered array of rules
  */
 async function exclude(rules, exclusions, exclusionsSources) {
     if (_.isEmpty(exclusions) && _.isEmpty(exclusionsSources)) {

src/transformations/include.js

@@ -9,7 +9,7 @@ const filterUtils = require('../filter');
  * @param {Array<String>} inclusions - array of inclusions to apply
  * @param {Array<String>} inclusionsSources - array of inclusions' sources
  * (can be a local or remote file)
- * @returns {Array<String>} filtered array of rules
+ * @returns {Promise<Array<String>>} filtered array of rules
  */
 async function include(rules, inclusions, inclusionsSources) {
     if (_.isEmpty(inclusions) && _.isEmpty(inclusionsSources)) {

src/transformations/insert-final-new-line.js

@@ -0,0 +1,17 @@
+const consola = require('consola');
+
+/**
+ * This is a very simple transformation that inserts a final new line.
+ *
+ * @param {Array<string>} lines - lines/rules to transform
+ * @returns {Array<string>} filtered lines/rules
+ */
+function insertFinalNewLine(lines) {
+    if (lines.length === 0 || (lines.length > 0 && lines[lines.length - 1].trim() !== '')) {
+        lines.push('');
+    }
+    consola.info('Final new line inserted');
+    return lines;
+}
+
+module.exports = insertFinalNewLine;

src/transformations/remove-empty-lines.js

@@ -0,0 +1,15 @@
+const consola = require('consola');
+
+/**
+ * This is a very simple transformation that removes empty lines.
+ *
+ * @param {Array<string>} lines - lines/rules to transform
+ * @returns {Array<string>} filtered lines/rules
+ */
+function removeEmptyLines(lines) {
+    const filtered = lines.filter((line) => line.trim().length);
+    consola.info(`Removed ${lines.length - filtered.length} empty lines`);
+    return filtered;
+}
+
+module.exports = removeEmptyLines;

src/transformations/transform.js

@@ -6,6 +6,9 @@ const include = require('./include');
 const deduplicate = require('./deduplicate');
 const compress = require('./compress');
 const invertAllow = require('./invertallow');
+const removeEmptyLines = require('./remove-empty-lines');
+const trimLines = require('./trim-lines');
+const insertFinalNewLine = require('./insert-final-new-line');
 
 /**
  * Enum with all available transformations
@@ -17,6 +20,9 @@ const TRANSFORMATIONS = Object.freeze({
     Validate: 'Validate',
     Deduplicate: 'Deduplicate',
     InvertAllow: 'InvertAllow',
+    RemoveEmptyLines: 'RemoveEmptyLines',
+    TrimLines: 'TrimLines',
+    InsertFinalNewLine: 'InsertFinalNewLine',
 });
 
 /**
@@ -25,7 +31,7 @@ const TRANSFORMATIONS = Object.freeze({
  * @param {Array<string>} rules - rules to transform
  * @param {*} configuration - transformation configuration.
  * @param {Array<string>} transformations - a list of transformations to apply to the rules.
- * @returns {Array<string>} rules after applying all transformations.
+ * @returns {Promise<Array<string>>} rules after applying all transformations.
  */
 async function transform(rules, configuration, transformations) {
     // If none specified -- apply all transformationss
@@ -59,6 +65,15 @@ async function transform(rules, configuration, transformations) {
     if (transformations.indexOf(TRANSFORMATIONS.Deduplicate) !== -1) {
         transformed = deduplicate(transformed);
     }
+    if (transformations.indexOf(TRANSFORMATIONS.RemoveEmptyLines) !== -1) {
+        transformed = removeEmptyLines(transformed);
+    }
+    if (transformations.indexOf(TRANSFORMATIONS.TrimLines) !== -1) {
+        transformed = trimLines(transformed);
+    }
+    if (transformations.indexOf(TRANSFORMATIONS.InsertFinalNewLine) !== -1) {
+        transformed = insertFinalNewLine(transformed);
+    }
     return transformed;
 }
 

src/transformations/trim-lines.js

@@ -0,0 +1,16 @@
+const consola = require('consola');
+const _ = require('lodash');
+
+/**
+ * This is a very simple transformation that removes leading and trailing spaces/tabs.
+ *
+ * @param {Array<string>} lines - lines/rules to transform
+ * @returns {Array<string>} filtered lines/rules
+ */
+function trimLines(lines) {
+    const transformed = lines.map((line) => _.trim(line, [' ', '\t']));
+    consola.info('Lines trimmed.');
+    return transformed;
+}
+
+module.exports = trimLines;

src/utils.js

@@ -16,7 +16,7 @@ function isURL(str) {
  * Downloads (or reads from the disk) the specified source
  *
  * @param {*} urlOrPath url or path to a file
- * @returns {String} contents of the files
+ * @returns {Promise<String>} contents of the files
  */
 async function download(urlOrPath) {
     let str = '';