HTML must be sanitized before parsing, to fixup any invalid characters

.travis.yml

@@ -1,3 +1,4 @@
 language: node_js
 node_js:
-  - "0.10"
+#  - "0.10"
+ - 8

Changes.md

@@ -14,6 +14,23 @@
  * Add warnings for parser errors?
  * Switch to semantic versioning?
 
+## Next major release:
+
+ * Event/error handling
+ * Error.code = 404, 'timeout', etc.
+ * Error.module = 'http', 'dom', etc.
+  * return true = retry, false = stop, anything else = continue
+ * Event for discontinued context/data
+ * Module system using osmosis.require and modules prefixed with `osmosis-`
+ * Way to trigger DOM
+ * Throw unhandled errors?
+ * `.while()` to do things more than once as long as they call next()
+
+## 0.1.5
+
+ * Fixed bug where .get() without `params` caused empty query string ('?')
+ * Preserve sort order for `.follow()` results within `.set()`
+
 ## 0.1.4
 
 #### `get`

Readme.md

@@ -84,9 +84,4 @@ For documentation and examples check out [https://rchipka.github.io/node-osmosis
 Please consider a donation if you depend on web scraping and Osmosis makes your job a bit easier.
 Your contribution allows me to spend more time making this the best web scraper for Node.
 
-### Donation offers:
-
- - $25 - A custom Osmosis scraper to extract the data you need efficiently and in as few lines of code as possible.
- - $25/month - Become a sponsor. Your company will be listed on this page. Priority support and bug fixes.
-
 [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif)](https://www.paypal.com/cgi-bin/webscr?item_name=node-osmosis&cmd=_donations&business=NAXMWBMWKUWUU)

index.js

@@ -179,6 +179,10 @@ Osmosis.prototype.request = function (url, opts, callback, tries) {
     this.requests++;
     this.queue.requests++;
     this.queue.push();
+
+    if (typeof opts.user_agent === 'function') {
+        opts.user_agent = opts.user_agent();
+    }
 
     request(url.method,
             url,
@@ -451,4 +455,6 @@ libxml.Element.prototype.find = function (selector) {
  * @param {data} data - The current data object.
  */
 
+Osmosis.libxmljs = libxml;
+
 module.exports = Osmosis;

lib/Command.js

@@ -156,14 +156,14 @@ Command.prototype.start = function (context, data) {
 
     data.ref();
 
-    return callback.call(this, context, data, function (c, d) {
+    return callback.call(this, context, data, function (c, d, index) {
         if (calledNext === true) {
             // If `next` is called more than once,
             // then we need to clone the data
-            next.start(c, d.clone().ref());
+            next.start(c, d.clone().setSortIndex(index).ref());
         } else {
             calledNext = true;
-            next.start(c, d);
+            next.start(c, d.setSortIndex(index));
         }
     }, function (err) {
         data.unref();
@@ -284,7 +284,7 @@ Command.prototype.setOpt = function (name, value) {
  * @private
  */
 
-Command.prototype.request = function (method, context, href, params, callback) {
+Command.prototype.request = function (method, context, href, params, callback, sortIndex) {
     var self     = this,
         length   = callback.length,
         instance = self.instance,
@@ -340,7 +340,7 @@ Command.prototype.request = function (method, context, href, params, callback) {
     url.method = method;
     url.params = params;
 
-    if (method === 'get') {
+    if (method === 'get' && params instanceof Object && params !== null) {
         for (key in params) {
             url.query[key] = params[key];
         }
@@ -386,6 +386,10 @@ Command.prototype.request = function (method, context, href, params, callback) {
                 '')
             );
 
+            if (document instanceof Object && document !== null) {
+                document._dataSortIndex = sortIndex;
+            }
+
             if (length === 1) {
                 callback(document);
             } else if (length === 2) {

lib/Data.js

@@ -5,7 +5,8 @@
  * @constructor Data
  * @param {object} [data] - Data object value
  * @param {object} [parent] - Parent Data object
- * @param {object} [index] - Index in the parent object
+ * @param {number} [index] - Index in the parent object
+ * @param {number} [sortIndex] - Sort order of object if coerced into array
  * @param {bool} [isArray] - Is the object an array?
  * @property {number} refs - Number of references
  * @property {number} clones - Number of clones
@@ -91,6 +92,7 @@ Data.prototype.setObject = function (object) {
 
 Data.prototype.next = function () {
     var clone = new Data(this.parent)
+                    .setSortIndex(this.getSortIndex())
                     .setIndex(this.getIndex())
                     .isArray(this.isArray());
 
@@ -130,7 +132,7 @@ Data.prototype.unref = function () {
  */
 
 Data.prototype.set = function (key, val) {
-    var object, currentVal;
+    var object, currentVal, sortKey;
 
     if (val === undefined) {
         return this;
@@ -222,9 +224,65 @@ Data.prototype.setIndex = function (index) {
     return this;
 };
 
+Data.prototype.setSortIndex = function (index) {
+    if (index !== undefined) {
+        this.sortIndex = index;
+    }
+
+    return this;
+};
+
+Data.prototype.getSortIndex = function () {
+    return this.sortIndex;
+}
+
+Data.prototype.sortKey = function (key, sortIndex) {
+    var object = this.getObject(),
+        currentVal = object[key],
+        sortArray;
+
+    if (!this.sortArray) {
+        this.sortArray = {};
+    }
+
+    sortArray = this.sortArray[key];
+
+    if (sortArray === undefined) {
+        if (currentVal instanceof Array && currentVal.length > 0) {
+            sortArray = new Array(currentVal.length);
+        } else {
+            sortArray = [sortIndex];
+        }
+
+        this.sortArray[key] = sortArray;
+    }
+
+    if (currentVal instanceof Array) {
+        var diff = currentVal.length - sortArray.length;
+
+        while (diff > 0) {
+            sortArray.push(sortIndex + (--diff));
+        }
+
+        object[key] = sortArray.map(function (v, i) {
+            return {
+                value: v,
+                index: i
+            };
+        }).sort(function (a, b) {
+            return a.value - b.value;
+        }).map(function (v, i) {
+            sortArray[i] = v.value;
+
+            return currentVal[v.index];
+        });
+    }
+}
+
 Data.prototype.merge = function (child) {
     var object = child.object,
-        index  = child.getIndex();
+        index  = child.getIndex(),
+        sortIndex = child.getSortIndex();
 
     if (object === undefined) {
         return;
@@ -233,10 +291,14 @@ Data.prototype.merge = function (child) {
     if (this.isArray() === true) {
         this.push(object);
     } else if (index !== undefined) {
-        this.set(child.getIndex(), object);
+        this.set(index, object);
     } else if (object instanceof Object) {
         this.extend(object);
     }
+
+    if (sortIndex !== undefined) {
+        this.sortKey(index, sortIndex);
+    }
 };
 
 Data.prototype.toArray = function () {

lib/Request.js

@@ -1,8 +1,9 @@
 'use strict';
 
-var needle = require('needle'),
-    URL    = require('url'),
-    libxml = require('libxmljs-dom');
+var needle       = require('needle'),
+    URL          = require('url'),
+    libxml       = require('libxmljs-dom'),
+    sanitizeHtml = require('sanitize-html');
 
 /**
  * Make an HTTP request.
@@ -12,13 +13,15 @@ var needle = require('needle'),
 
 function Request(method, url, params, opts, tries, callback) {
     var location = url;
-
     return needle.request(method,
-                          url.href,
+                          encodeURI(url.href),
                           params,
                           opts,
                           function (err, res, data) {
-        var document;
+
+        if (!(url.params instanceof Object) || url.params === null) {
+            url.params = url.query;
+        }
 
         if (err !== null) {
             callback(err.message);
@@ -28,7 +31,8 @@ function Request(method, url, params, opts, tries, callback) {
         if (opts.ignore_http_errors !== true &&
             res                     !== undefined &&
             res.statusCode          >=  400   &&
-            res.statusCode          <=  500) {
+            res.statusCode          <=  500
+        ) {
             // HTTP error
             callback(res.statusCode + ' ' + res.statusMessage);
             return;
@@ -39,49 +43,68 @@ function Request(method, url, params, opts, tries, callback) {
             return;
         }
 
-        if (opts.process_response !== undefined) {
-            document = opts.process_response(data);
-        } else {
-            document = data;
-        }
+        function next(document) {
+            if (opts.parse === false) {
+                callback(null, res, document);
+                return;
+            }
 
-        if (opts.parse !== false) {
             document = libxml.parseHtml(document,
-                                        { baseUrl: location.href, huge: true });
-        }
+                { baseUrl: location.href, huge: true });
 
-        if (document === null) {
-            callback('Couldn\'t parse response');
-            return;
-        }
+            if (document === null) {
+                callback('Couldn\'t parse response');
+                return;
+            }
 
-        if (document.errors[0] !== undefined &&
-            document.errors[0].code === 4) {
-            callback('Document is empty');
-            return;
-        }
+            if (document.errors[0] !== undefined &&
+                document.errors[0].code === 4) {
+                callback('Document is empty');
+                return;
+            }
 
-        if (document.root() === null) {
-            callback('Document has no root');
-            return;
-        }
+            if (document.root() === null) {
+                callback('Document has no root');
+                return;
+            }
+
+            location.headers    = res.req._headers;
+            location.proxy      = opts.proxy;
+            location.user_agent = opts.user_agent;
 
-        location.headers    = res.req._headers;
-        location.proxy      = opts.proxy;
-        location.user_agent = opts.user_agent;
+            document.location = location;
+            document.request  = location;
 
-        document.location = location;
-        document.request  = location;
+            setResponseMeta(document, res, data.length);
+            setCookies(document, res.cookies);
+            setCookies(document, opts.cookies);
 
-        setResponseMeta(document, res, data.length);
-        setCookies(document, res.cookies);
-        setCookies(document, opts.cookies);
+            if (opts.keep_data === true) {
+                document.response.data = data;
+            }
 
-        if (opts.keep_data === true) {
-            document.response.data = data;
+            callback(null, res, document);
+        }
+
+        if (
+            opts.process_response !== undefined &&
+            typeof opts.process_response === 'function'
+        ) {
+            if (opts.process_response.length > 2) {
+                opts.process_response(data, res, next, callback);
+                return;
+            }
+
+            next(opts.process_response(data, res));
+        } else {
+            const cleanData = sanitizeHtml(data, {
+                allowedTags: false,
+                allowedAttributes: false,
+                selfClosing: [],
+            });
+            next(cleanData);
         }
 
-        callback(null, res, document);
     })
     .on('redirect', function (href) {
         extend(location, URL.parse(URL.resolve(location.href, href)));

lib/commands/find.js

@@ -51,7 +51,7 @@ var Find = function (context, data, next, done) {
         node = nodes[i];
         node.last = (length - 1 === i);
         node.index = i;
-        next(node, data);
+        next(node, data, i);
     }
 
     done();

lib/commands/follow.js

@@ -18,7 +18,7 @@ module.exports.follow = function (context, data, next, done) {
         i = 0, queue = 0, length, node, url,
         requestDone = function (err, document) {
             if (err === null) {
-                next(document, data);
+                next(document, data, document._dataSortIndex);
             }
 
             if (--queue === 0) {
@@ -52,10 +52,11 @@ module.exports.follow = function (context, data, next, done) {
 
             self.log("url: " + url);
             self.request('get',
-                        context,
+                        node,
                         url,
                         null,
-                        requestDone);
+                        requestDone,
+                        i);
         }
     }