From e1c601ea57ffc5b3c99297e15488297d805cfbf1 Mon Sep 17 00:00:00 2001 From: Adam Kariv Date: Tue, 12 Dec 2017 12:46:22 +0200 Subject: [PATCH] [openspending/openspending#1332] Trim file contents before sending to goodtables --- app/front/scripts/services/validation.js | 31 +- package-lock.json | 810 ++++++++++++++++++++++- 2 files changed, 832 insertions(+), 9 deletions(-) diff --git a/app/front/scripts/services/validation.js b/app/front/scripts/services/validation.js index 0a134be..40206c5 100644 --- a/app/front/scripts/services/validation.js +++ b/app/front/scripts/services/validation.js @@ -15,8 +15,35 @@ angular.module('Application') function($q) { return { validateResource: function(source) { - if (source.blob) source = source.blob; - return goodtables.validate(source, goodtablesOptions); + var data; + if (source.data) { + data = source.data; + var byteView = new Uint8Array(data); + var byteLimit = 64 * 1024; + if (byteView.length > byteLimit) { + var bl = 64 * 1024; + var cutoff = -1; + if (byteView.indexOf) { + cutoff = byteView.indexOf(10, bl - 1024); + } else { + for (var i = bl - 1024; i < bl; i++) { + if (byteView[i] === 10) { + cutoff = i; + } + } + } + if (cutoff === -1 || cutoff > bl) { + byteView = byteView.subarray(0, bl); + } else { + byteView = byteView.subarray(0, cutoff); + } + } + data = new File([byteView], source.blob.name, {type: source.blob.type}); + } else { + data = source; + } + + return goodtables.validate(data, goodtablesOptions); }, validateRequiredConcepts: function(errors, resources) { var hasConcept = function(prefix) { diff --git a/package-lock.json b/package-lock.json index 4d8ad98..bc59b47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -564,7 +564,7 @@ }, "chai": { "version": "3.5.0", - "resolved": "http://registry.npmjs.org/chai/-/chai-3.5.0.tgz", + "resolved": "https://registry.npmjs.org/chai/-/chai-3.5.0.tgz", "integrity": "sha1-TQJjewZ/6Vi9v906QOxW/vc3Mkc=", "dev": true, "requires": { @@ -593,6 +593,7 @@ "requires": { "anymatch": "1.3.2", "async-each": "1.0.1", + "fsevents": "1.1.3", "glob-parent": "2.0.0", "inherits": "2.0.3", "is-binary-path": "1.0.1", @@ -1845,6 +1846,795 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, + "fsevents": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.1.3.tgz", + "integrity": "sha512-WIr7iDkdmdbxu/Gh6eKEZJL6KPE74/5MEsf2whTOFNxbIoIixogroLdKYqB6FDav4Wavh/lZdzzd3b2KxIXC5Q==", + "optional": true, + "requires": { + "nan": "2.8.0", + "node-pre-gyp": "0.6.39" + }, + "dependencies": { + "abbrev": { + "version": "1.1.0", + "bundled": true, + "optional": true + }, + "ajv": { + "version": "4.11.8", + "bundled": true, + "optional": true, + "requires": { + "co": "4.6.0", + "json-stable-stringify": "1.0.1" + } + }, + "ansi-regex": { + "version": "2.1.1", + "bundled": true + }, + "aproba": { + "version": "1.1.1", + "bundled": true, + "optional": true + }, + "are-we-there-yet": { + "version": "1.1.4", + "bundled": true, + "optional": true, + "requires": { + "delegates": "1.0.0", + "readable-stream": "2.2.9" + } + }, + "asn1": { + "version": "0.2.3", + "bundled": true, + "optional": true + }, + "assert-plus": { + "version": "0.2.0", + "bundled": true, + "optional": true + }, + "asynckit": { + "version": "0.4.0", + "bundled": true, + "optional": true + }, + "aws-sign2": { + "version": "0.6.0", + "bundled": true, + "optional": true + }, + "aws4": { + "version": "1.6.0", + "bundled": true, + "optional": true + }, + "balanced-match": { + "version": "0.4.2", + "bundled": true + }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "bundled": true, + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "block-stream": { + "version": "0.0.9", + "bundled": true, + "requires": { + "inherits": "2.0.3" + } + }, + "boom": { + "version": "2.10.1", + "bundled": true, + "requires": { + "hoek": "2.16.3" + } + }, + "brace-expansion": { + "version": "1.1.7", + "bundled": true, + "requires": { + "balanced-match": "0.4.2", + "concat-map": "0.0.1" + } + }, + "buffer-shims": { + "version": "1.0.0", + "bundled": true + }, + "caseless": { + "version": "0.12.0", + "bundled": true, + "optional": true + }, + "co": { + "version": "4.6.0", + "bundled": true, + "optional": true + }, + "code-point-at": { + "version": "1.1.0", + "bundled": true + }, + "combined-stream": { + "version": "1.0.5", + "bundled": true, + "requires": { + "delayed-stream": "1.0.0" + } + }, + "concat-map": { + "version": "0.0.1", + "bundled": true + }, + "console-control-strings": { + "version": "1.1.0", + "bundled": true + }, + "core-util-is": { + "version": "1.0.2", + "bundled": true + }, + "cryptiles": { + "version": "2.0.5", + "bundled": true, + "requires": { + "boom": "2.10.1" + } + }, + "dashdash": { + "version": "1.14.1", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "debug": { + "version": "2.6.8", + "bundled": true, + "optional": true, + "requires": { + "ms": "2.0.0" + } + }, + "deep-extend": { + "version": "0.4.2", + "bundled": true, + "optional": true + }, + "delayed-stream": { + "version": "1.0.0", + "bundled": true + }, + "delegates": { + "version": "1.0.0", + "bundled": true, + "optional": true + }, + "detect-libc": { + "version": "1.0.2", + "bundled": true, + "optional": true + }, + "ecc-jsbn": { + "version": "0.1.1", + "bundled": true, + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "extend": { + "version": "3.0.1", + "bundled": true, + "optional": true + }, + "extsprintf": { + "version": "1.0.2", + "bundled": true + }, + "forever-agent": { + "version": "0.6.1", + "bundled": true, + "optional": true + }, + "form-data": { + "version": "2.1.4", + "bundled": true, + "optional": true, + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.5", + "mime-types": "2.1.15" + } + }, + "fs.realpath": { + "version": "1.0.0", + "bundled": true + }, + "fstream": { + "version": "1.0.11", + "bundled": true, + "requires": { + "graceful-fs": "4.1.11", + "inherits": "2.0.3", + "mkdirp": "0.5.1", + "rimraf": "2.6.1" + } + }, + "fstream-ignore": { + "version": "1.0.5", + "bundled": true, + "optional": true, + "requires": { + "fstream": "1.0.11", + "inherits": "2.0.3", + "minimatch": "3.0.4" + } + }, + "gauge": { + "version": "2.7.4", + "bundled": true, + "optional": true, + "requires": { + "aproba": "1.1.1", + "console-control-strings": "1.1.0", + "has-unicode": "2.0.1", + "object-assign": "4.1.1", + "signal-exit": "3.0.2", + "string-width": "1.0.2", + "strip-ansi": "3.0.1", + "wide-align": "1.1.2" + } + }, + "getpass": { + "version": "0.1.7", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "1.0.0" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "glob": { + "version": "7.1.2", + "bundled": true, + "requires": { + "fs.realpath": "1.0.0", + "inflight": "1.0.6", + "inherits": "2.0.3", + "minimatch": "3.0.4", + "once": "1.4.0", + "path-is-absolute": "1.0.1" + } + }, + "graceful-fs": { + "version": "4.1.11", + "bundled": true + }, + "har-schema": { + "version": "1.0.5", + "bundled": true, + "optional": true + }, + "har-validator": { + "version": "4.2.1", + "bundled": true, + "optional": true, + "requires": { + "ajv": "4.11.8", + "har-schema": "1.0.5" + } + }, + "has-unicode": { + "version": "2.0.1", + "bundled": true, + "optional": true + }, + "hawk": { + "version": "3.1.3", + "bundled": true, + "requires": { + "boom": "2.10.1", + "cryptiles": "2.0.5", + "hoek": "2.16.3", + "sntp": "1.0.9" + } + }, + "hoek": { + "version": "2.16.3", + "bundled": true + }, + "http-signature": { + "version": "1.1.1", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "0.2.0", + "jsprim": "1.4.0", + "sshpk": "1.13.0" + } + }, + "inflight": { + "version": "1.0.6", + "bundled": true, + "requires": { + "once": "1.4.0", + "wrappy": "1.0.2" + } + }, + "inherits": { + "version": "2.0.3", + "bundled": true + }, + "ini": { + "version": "1.3.4", + "bundled": true, + "optional": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "bundled": true, + "requires": { + "number-is-nan": "1.0.1" + } + }, + "is-typedarray": { + "version": "1.0.0", + "bundled": true, + "optional": true + }, + "isarray": { + "version": "1.0.0", + "bundled": true + }, + "isstream": { + "version": "0.1.2", + "bundled": true, + "optional": true + }, + "jodid25519": { + "version": "1.0.2", + "bundled": true, + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, + "jsbn": { + "version": "0.1.1", + "bundled": true, + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "bundled": true, + "optional": true + }, + "json-stable-stringify": { + "version": "1.0.1", + "bundled": true, + "optional": true, + "requires": { + "jsonify": "0.0.0" + } + }, + "json-stringify-safe": { + "version": "5.0.1", + "bundled": true, + "optional": true + }, + "jsonify": { + "version": "0.0.0", + "bundled": true, + "optional": true + }, + "jsprim": { + "version": "1.4.0", + "bundled": true, + "optional": true, + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.0.2", + "json-schema": "0.2.3", + "verror": "1.3.6" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "mime-db": { + "version": "1.27.0", + "bundled": true + }, + "mime-types": { + "version": "2.1.15", + "bundled": true, + "requires": { + "mime-db": "1.27.0" + } + }, + "minimatch": { + "version": "3.0.4", + "bundled": true, + "requires": { + "brace-expansion": "1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "bundled": true + }, + "mkdirp": { + "version": "0.5.1", + "bundled": true, + "requires": { + "minimist": "0.0.8" + } + }, + "ms": { + "version": "2.0.0", + "bundled": true, + "optional": true + }, + "node-pre-gyp": { + "version": "0.6.39", + "bundled": true, + "optional": true, + "requires": { + "detect-libc": "1.0.2", + "hawk": "3.1.3", + "mkdirp": "0.5.1", + "nopt": "4.0.1", + "npmlog": "4.1.0", + "rc": "1.2.1", + "request": "2.81.0", + "rimraf": "2.6.1", + "semver": "5.3.0", + "tar": "2.2.1", + "tar-pack": "3.4.0" + } + }, + "nopt": { + "version": "4.0.1", + "bundled": true, + "optional": true, + "requires": { + "abbrev": "1.1.0", + "osenv": "0.1.4" + } + }, + "npmlog": { + "version": "4.1.0", + "bundled": true, + "optional": true, + "requires": { + "are-we-there-yet": "1.1.4", + "console-control-strings": "1.1.0", + "gauge": "2.7.4", + "set-blocking": "2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "bundled": true + }, + "oauth-sign": { + "version": "0.8.2", + "bundled": true, + "optional": true + }, + "object-assign": { + "version": "4.1.1", + "bundled": true, + "optional": true + }, + "once": { + "version": "1.4.0", + "bundled": true, + "requires": { + "wrappy": "1.0.2" + } + }, + "os-homedir": { + "version": "1.0.2", + "bundled": true, + "optional": true + }, + "os-tmpdir": { + "version": "1.0.2", + "bundled": true, + "optional": true + }, + "osenv": { + "version": "0.1.4", + "bundled": true, + "optional": true, + "requires": { + "os-homedir": "1.0.2", + "os-tmpdir": "1.0.2" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "bundled": true + }, + "performance-now": { + "version": "0.2.0", + "bundled": true, + "optional": true + }, + "process-nextick-args": { + "version": "1.0.7", + "bundled": true + }, + "punycode": { + "version": "1.4.1", + "bundled": true, + "optional": true + }, + "qs": { + "version": "6.4.0", + "bundled": true, + "optional": true + }, + "rc": { + "version": "1.2.1", + "bundled": true, + "optional": true, + "requires": { + "deep-extend": "0.4.2", + "ini": "1.3.4", + "minimist": "1.2.0", + "strip-json-comments": "2.0.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "bundled": true, + "optional": true + } + } + }, + "readable-stream": { + "version": "2.2.9", + "bundled": true, + "requires": { + "buffer-shims": "1.0.0", + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "1.0.7", + "string_decoder": "1.0.1", + "util-deprecate": "1.0.2" + } + }, + "request": { + "version": "2.81.0", + "bundled": true, + "optional": true, + "requires": { + "aws-sign2": "0.6.0", + "aws4": "1.6.0", + "caseless": "0.12.0", + "combined-stream": "1.0.5", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.1.4", + "har-validator": "4.2.1", + "hawk": "3.1.3", + "http-signature": "1.1.1", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.15", + "oauth-sign": "0.8.2", + "performance-now": "0.2.0", + "qs": "6.4.0", + "safe-buffer": "5.0.1", + "stringstream": "0.0.5", + "tough-cookie": "2.3.2", + "tunnel-agent": "0.6.0", + "uuid": "3.0.1" + } + }, + "rimraf": { + "version": "2.6.1", + "bundled": true, + "requires": { + "glob": "7.1.2" + } + }, + "safe-buffer": { + "version": "5.0.1", + "bundled": true + }, + "semver": { + "version": "5.3.0", + "bundled": true, + "optional": true + }, + "set-blocking": { + "version": "2.0.0", + "bundled": true, + "optional": true + }, + "signal-exit": { + "version": "3.0.2", + "bundled": true, + "optional": true + }, + "sntp": { + "version": "1.0.9", + "bundled": true, + "requires": { + "hoek": "2.16.3" + } + }, + "sshpk": { + "version": "1.13.0", + "bundled": true, + "optional": true, + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jodid25519": "1.0.2", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + }, + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "bundled": true, + "optional": true + } + } + }, + "string-width": { + "version": "1.0.2", + "bundled": true, + "requires": { + "code-point-at": "1.1.0", + "is-fullwidth-code-point": "1.0.0", + "strip-ansi": "3.0.1" + } + }, + "string_decoder": { + "version": "1.0.1", + "bundled": true, + "requires": { + "safe-buffer": "5.0.1" + } + }, + "stringstream": { + "version": "0.0.5", + "bundled": true, + "optional": true + }, + "strip-ansi": { + "version": "3.0.1", + "bundled": true, + "requires": { + "ansi-regex": "2.1.1" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "bundled": true, + "optional": true + }, + "tar": { + "version": "2.2.1", + "bundled": true, + "requires": { + "block-stream": "0.0.9", + "fstream": "1.0.11", + "inherits": "2.0.3" + } + }, + "tar-pack": { + "version": "3.4.0", + "bundled": true, + "optional": true, + "requires": { + "debug": "2.6.8", + "fstream": "1.0.11", + "fstream-ignore": "1.0.5", + "once": "1.4.0", + "readable-stream": "2.2.9", + "rimraf": "2.6.1", + "tar": "2.2.1", + "uid-number": "0.0.6" + } + }, + "tough-cookie": { + "version": "2.3.2", + "bundled": true, + "optional": true, + "requires": { + "punycode": "1.4.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "bundled": true, + "optional": true, + "requires": { + "safe-buffer": "5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "bundled": true, + "optional": true + }, + "uid-number": { + "version": "0.0.6", + "bundled": true, + "optional": true + }, + "util-deprecate": { + "version": "1.0.2", + "bundled": true + }, + "uuid": { + "version": "3.0.1", + "bundled": true, + "optional": true + }, + "verror": { + "version": "1.3.6", + "bundled": true, + "optional": true, + "requires": { + "extsprintf": "1.0.2" + } + }, + "wide-align": { + "version": "1.1.2", + "bundled": true, + "optional": true, + "requires": { + "string-width": "1.0.2" + } + }, + "wrappy": { + "version": "1.0.2", + "bundled": true + } + } + }, "gaze": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/gaze/-/gaze-0.5.2.tgz", @@ -2121,7 +2911,7 @@ }, "gulp": { "version": "3.9.1", - "resolved": "http://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz", + "resolved": "https://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz", "integrity": "sha1-VxzkWSjdQK9lFPxAEYZgFsE4RbQ=", "dev": true, "requires": { @@ -2235,7 +3025,7 @@ }, "gulp-rename": { "version": "1.2.2", - "resolved": "http://registry.npmjs.org/gulp-rename/-/gulp-rename-1.2.2.tgz", + "resolved": "https://registry.npmjs.org/gulp-rename/-/gulp-rename-1.2.2.tgz", "integrity": "sha1-OtRCh2PwXidk3sHGfYaNsnVoeBc=", "dev": true }, @@ -2378,7 +3168,7 @@ "readable-stream": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", - "integrity": "sha1-No8lEtefnUb9/HE0mueHi7weuVw=", + "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", "dev": true, "requires": { "core-util-is": "1.0.2", @@ -3511,7 +4301,7 @@ }, "marked": { "version": "0.3.6", - "resolved": "http://registry.npmjs.org/marked/-/marked-0.3.6.tgz", + "resolved": "https://registry.npmjs.org/marked/-/marked-0.3.6.tgz", "integrity": "sha1-ssbGGPzOzk74bE/Gy4p8v1rtqNc=" }, "media-typer": { @@ -3650,7 +4440,7 @@ "dependencies": { "debug": { "version": "2.2.0", - "resolved": "http://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", "dev": true, "requires": { @@ -3722,6 +4512,12 @@ "integrity": "sha1-j7+rsKmKJT0xhDMfno3rc3L6xsA=", "dev": true }, + "nan": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.8.0.tgz", + "integrity": "sha1-7XFfP+neArV6XmJS2QqWZ14fCFo=", + "optional": true + }, "natives": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/natives/-/natives-1.1.0.tgz", @@ -5526,7 +6322,7 @@ "dependencies": { "depd": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/depd/-/depd-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz", "integrity": "sha1-4b2Cxqq2ztlluXuIsX7T5SjKGMM=" } }