From 96c193816635cdaa635070f4448360a46578c5e3 Mon Sep 17 00:00:00 2001 From: Jose De Gouveia Date: Tue, 22 Nov 2022 14:09:20 +0100 Subject: [PATCH] Updated public action #90, fixed #91 not getting correct reqOptions, fixed request broken after getTotalSize is called and there is a http to https redirect --- .github/workflows/npmpublish.yml | 4 +-- .github/workflows/test.yml | 8 +++--- .gitignore | 2 +- .gitlab-ci.yml | 2 +- .travis.yml | 2 +- README.md | 6 ++-- example/index.js | 1 + package.json | 2 +- src/index.js | 47 ++++++++++++++++++-------------- test/index.spec.js | 41 ++++++++++++++++++++++++++-- types/index.d.ts | 5 +++- 11 files changed, 84 insertions(+), 36 deletions(-) diff --git a/.github/workflows/npmpublish.yml b/.github/workflows/npmpublish.yml index 51d2420..3d8badb 100644 --- a/.github/workflows/npmpublish.yml +++ b/.github/workflows/npmpublish.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: 14 + node-version: 18 - run: npm install --no-optional - run: npm test @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: 14 + node-version: 18 registry-url: https://registry.npmjs.org/ - run: npm install --no-optional - run: npm run build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a532f1b..1357b22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,16 +2,16 @@ name: Node.js CI on: push: - branches: [ main, dev ] + branches: [main, dev] pull_request: - branches: [ main, dev ] + branches: [main, dev] jobs: build: runs-on: ubuntu-latest strategy: matrix: - node-version: [14.x, 15.x, 16.x, 19.x] + node-version: [14.x, 16.x, 19.x] steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} @@ -19,4 +19,4 @@ jobs: with: node-version: ${{ matrix.node-version }} - run: npm install --no-optional - - run: npm test \ No newline at end of file + - run: npm test diff --git a/.gitignore b/.gitignore index 0f1ab94..f157dea 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ package-lock.json example/* !example/*.js dist -sandbox.js \ No newline at end of file +*sandbox.js \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9ce865..b412507 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: node:14 +image: node:18 cache: paths: diff --git a/.travis.yml b/.travis.yml index a6be8e0..f6a76ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: node_js node_js: - - "14.18.3" \ No newline at end of file + - "18.12.1" \ No newline at end of file diff --git a/README.md b/README.md index 6f18374..797e8f6 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ [![NPM Version](https://img.shields.io/npm/v/node-downloader-helper.svg?style=flat-square "npm version")](https://www.npmjs.com/package/node-downloader-helper) ![npm](https://img.shields.io/npm/dw/node-downloader-helper?style=flat-square "npm download") ![GitHub Actions Build](https://github.com/hgouveia/node-downloader-helper/actions/workflows/test.yml/badge.svg "GitHub Actions Build") -[![Build Status](https://img.shields.io/travis/hgouveia/node-downloader-helper/master.svg?style=flat-square "Build Status")](https://travis-ci.org/hgouveia/node-downloader-helper) [![Windows Build Status](https://img.shields.io/appveyor/ci/hgouveia/node-downloader-helper/master.svg?label=windows&style=flat-square "Windows Build Status")](https://ci.appveyor.com/project/hgouveia/node-downloader-helper) [![Join the chat at https://gitter.im/node-downloader-helper/Lobby](https://badges.gitter.im/node-downloader-helper/Lobby.svg)](https://gitter.im/node-downloader-helper/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fhgouveia%2Fnode-downloader-helper.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fhgouveia%2Fnode-downloader-helper?ref=badge_shield) @@ -120,7 +119,7 @@ for `httpsRequestOptions` the available options are detailed in here https://nod | stop | stop the downloading and remove the file | | pipe | `readable.pipe(stream.Readable, options) : stream.Readable` | | unpipe | `(stream)` if not stream is not specified, then all pipes are detached. | -| updateOptions | `(options)` updates the options, can be use on pause/resume events | +| updateOptions | `(options, url)` updates the options, can be use on pause/resume events | | getStats | returns `stats` from the current download, these are the same `stats` sent via progress event | | getTotalSize | gets the total file size from the server | | getDownloadPath | gets the full path where the file will be downloaded (available after the start phase) | @@ -160,9 +159,10 @@ dl.resumeFromFile(prevState.filePath, prevState).catch(err => console.error(err) | error | Emitted when there is any error `callback(error)` | | timeout | Emitted when the underlying socket times out from inactivity. | | pause | Emitted when the .pause method is called | -| resume | Emitted when the .resume method is called `callback(isResume)` | | stop | Emitted when the .stop method is called | +| resume | Emitted when the .resume method is called `callback(isResume)` | | renamed | Emitted when '(number)' is appended to the end of file, this requires `override:false` opt, `callback(filePaths)` | +| redirected | Emitted when an url redirect happened `callback(newUrl, oldUrl)` NOTE: this will be triggered during getTotalSize() as well | | stateChanged | Emitted when the state changes `callback(state)` | | warning | Emitted when an error occurs that was not thrown intentionally `callback(err: Error)` | diff --git a/example/index.js b/example/index.js index 361a208..00ad5ac 100644 --- a/example/index.js +++ b/example/index.js @@ -72,6 +72,7 @@ dl }) .on('stateChanged', state => console.log('State: ', state)) .on('renamed', filePaths => console.log('File Renamed to: ', filePaths.fileName)) + .on('redirected', (newUrl, oldUrl) => console.log(`Redirect from '${newUrl}' => '${oldUrl}'`)) .on('progress', stats => { const progress = stats.progress.toFixed(1); const speed = byteHelper(stats.speed); diff --git a/package.json b/package.json index e83de3e..dd32de1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-downloader-helper", - "version": "2.1.4", + "version": "2.1.5", "description": "A simple http file downloader for node.js", "main": "./dist/index.js", "types": "./types/index.d.ts", diff --git a/src/index.js b/src/index.js index 3403026..ea12ecd 100644 --- a/src/index.js +++ b/src/index.js @@ -36,7 +36,6 @@ export class DownloaderHelper extends EventEmitter { this.url = this.requestURL = url.trim(); this.state = DH_STATES.IDLE; - this.__agent = null; this.__defaultOpts = { body: null, retry: false, // { maxRetries: 3, delay: 3000 } @@ -158,7 +157,7 @@ export class DownloaderHelper extends EventEmitter { this.__setState(this.__states.RESUMED); if (this.__isResumable) { this.__isResumed = true; - this.__options['headers']['range'] = `bytes=${this.__downloaded}-`; + this.__reqOptions['headers']['range'] = `bytes=${this.__downloaded}-`; } this.emit('resume', this.__isResumed); return this.__start(); @@ -269,9 +268,10 @@ export class DownloaderHelper extends EventEmitter { * Updates the options, can be use on pause/resume events * * @param {Object} [options={}] + * @param {String} [url=''] * @memberof DownloaderHelper */ - updateOptions(options) { + updateOptions(options, url = '') { this.__opts = Object.assign({}, this.__opts, options); this.__headers = this.__opts.headers; @@ -285,8 +285,9 @@ export class DownloaderHelper extends EventEmitter { this.__opts.progressThrottle = this.__defaultOpts.progressThrottle; } + this.url = url || this.url; + this.__reqOptions = this.__getReqOptions(this.__opts.method, this.url, this.__opts.headers); this.__initProtocol(this.url); - this.__options = this.__getOptions(this.__opts.method, this.url, this.__opts.headers); } /** @@ -328,19 +329,24 @@ export class DownloaderHelper extends EventEmitter { * @memberof DownloaderHelper */ getTotalSize() { - const headers = Object.assign({}, this.__headers); - if (headers.hasOwnProperty('range')) { - delete headers['range']; - } return new Promise((resolve, reject) => { + const getReqOptions = (url) => { + this.__initProtocol(url); + const headers = Object.assign({}, this.__headers); + if (headers.hasOwnProperty('range')) { + delete headers['range']; + } + const reqOptions = this.__getReqOptions('HEAD', url, headers); + return Object.assign({}, this.__reqOptions, reqOptions); + }; const getRequest = (url, options) => { const req = this.__protocol.request(options, response => { if (this.__isRequireRedirect(response)) { const redirectedURL = /^https?:\/\//.test(response.headers.location) ? response.headers.location : new URL(response.headers.location, url).href; - this.__initProtocol(redirectedURL); - return getRequest(redirectedURL, this.__getOptions('HEAD', redirectedURL, headers)); + this.emit('redirected', redirectedURL, url); + return getRequest(redirectedURL, getReqOptions(redirectedURL)); } if (response.statusCode !== 200) { return reject(new Error(`Response status was ${response.statusCode}`)); @@ -355,7 +361,7 @@ export class DownloaderHelper extends EventEmitter { req.on('uncaughtException', (err) => reject(err)); req.end(); }; - getRequest(this.url, this.__getOptions('HEAD', this.url, headers)); + getRequest(this.url, getReqOptions(this.url)); }); } @@ -394,7 +400,7 @@ export class DownloaderHelper extends EventEmitter { this.__total = state.total || total; this.__fileName = state.fileName || name; this.__downloaded = state.downloaded || this.__getFilesizeInBytes(this.__filePath); - this.__options['headers']['range'] = `bytes=${this.__downloaded}-`; + this.__reqOptions['headers']['range'] = `bytes=${this.__downloaded}-`; this.__isResumed = true; this.__isResumable = true; this.__setState(this.__states.RESUMED); @@ -411,6 +417,7 @@ export class DownloaderHelper extends EventEmitter { this.state !== this.__states.RESUMED) { this.emit('start'); this.__setState(this.__states.STARTED); + this.__initProtocol(this.url); } // Start the Download @@ -453,7 +460,7 @@ export class DownloaderHelper extends EventEmitter { * @memberof DownloaderHelper */ __downloadRequest(resolve, reject) { - return this.__protocol.request(this.__options, response => { + return this.__protocol.request(this.__reqOptions, response => { this.__response = response; //Stats @@ -469,6 +476,7 @@ export class DownloaderHelper extends EventEmitter { : new URL(response.headers.location, this.url).href; this.__isRedirected = true; this.__initProtocol(redirectedURL); + this.emit('redirected', redirectedURL, this.url); return this.__start(); } @@ -951,7 +959,7 @@ export class DownloaderHelper extends EventEmitter { * @returns {Object} * @memberof DownloaderHelper */ - __getOptions(method, url, headers = {}) { + __getReqOptions(method, url, headers = {}) { const urlParse = new URL(url); const options = { protocol: urlParse.protocol, @@ -959,7 +967,6 @@ export class DownloaderHelper extends EventEmitter { port: urlParse.port, path: urlParse.pathname + urlParse.search, method, - agent: this.__agent, }; if (headers) { @@ -1041,17 +1048,17 @@ export class DownloaderHelper extends EventEmitter { * @memberof DownloaderHelper */ __initProtocol(url) { - const defaultOpts = this.__getOptions(this.__opts.method, url, this.__headers); + const defaultOpts = this.__getReqOptions(this.__opts.method, url, this.__headers); this.requestURL = url; if (url.indexOf('https://') > -1) { this.__protocol = https; - this.__agent = new https.Agent({ keepAlive: false }); - this.__options = Object.assign({}, defaultOpts, this.__opts.httpsRequestOptions); + defaultOpts.agent = new https.Agent({ keepAlive: false }); + this.__reqOptions = Object.assign({}, defaultOpts, this.__opts.httpsRequestOptions); } else { this.__protocol = http; - this.__agent = new http.Agent({ keepAlive: false }); - this.__options = Object.assign({}, defaultOpts, this.__opts.httpRequestOptions); + defaultOpts.agent = new http.Agent({ keepAlive: false }); + this.__reqOptions = Object.assign({}, defaultOpts, this.__opts.httpRequestOptions); } } diff --git a/test/index.spec.js b/test/index.spec.js index cfcd451..6491dec 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -313,12 +313,12 @@ describe('DownloaderHelper', function () { }); - describe('__getOptions', () => { + describe('__getReqOptions', () => { it("it should return the correct parsed options", function () { const dl = new DownloaderHelper('https://www.google.com/search?q=javascript', __dirname, { headers: { 'user-agent': 'my-user-agent' } }); - const options = dl.__getOptions(dl.__opts.method, dl.url, dl.__opts.headers); + const options = dl.__getReqOptions(dl.__opts.method, dl.url, dl.__opts.headers); expect(options.protocol).to.be.equal('https:'); expect(options.host).to.be.equal('www.google.com'); expect(options.port).to.be.equal(''); @@ -330,6 +330,43 @@ describe('DownloaderHelper', function () { }); }) + describe('__initProtocol', function () { + it("protocol should be http", function () { + const dl = new DownloaderHelper(downloadURL.replace('https:', 'http:'), __dirname); + expect(dl.__protocol.STATUS_CODES).to.be.not.undefined; + }); + + it("protocol should be https", function () { + // NOTE: STATUS_CODES property seems to be available only in http module + const dl = new DownloaderHelper(downloadURL, __dirname); + expect(dl.__protocol.STATUS_CODES).to.be.undefined; + }); + + it("protocol should has https Agent", function () { + const dl = new DownloaderHelper(downloadURL, __dirname); + expect(dl.__reqOptions.agent).to.be.not.undefined; + }); + + it("protocol should has http Agent", function () { + const dl = new DownloaderHelper(downloadURL.replace('https:', 'http:'), __dirname); + expect(dl.__reqOptions.agent).to.be.not.undefined; + }); + + it("protocol should has custom http Agent", function () { + const dl = new DownloaderHelper(downloadURL, __dirname, { + httpsRequestOptions: { agent: 'myCustomAgent' } + }); + expect(dl.__reqOptions.agent).to.be.equal('myCustomAgent'); + }); + + it("protocol should has custom https Agent", function () { + const dl = new DownloaderHelper(downloadURL.replace('https:', 'http:'), __dirname, { + httpRequestOptions: { agent: 'myCustomAgent' } + }); + expect(dl.__reqOptions.agent).to.be.equal('myCustomAgent'); + }); + }); + describe('download', function () { it("if the content-length is not present when the download starts, it should return null as totalSize", function (done) { fs.createWriteStream.mockReturnValue({ on: jest.fn() }); diff --git a/types/index.d.ts b/types/index.d.ts index 9668ac7..211431b 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -98,6 +98,8 @@ interface DownloadEvents { stop: () => any; /** Emitted when '(number)' is appended to the end of file, this requires override:false opt, callback(filePaths) */ renamed: (stats: FileRenamedStats) => any; + /** Emitted when an url redirect happened `callback(newUrl, oldUrl)` NOTE: this will be triggered during getTotalSize() as well */ + redirected: (newUrl: string, oldUrl: string) => any; /** Emitted when the state changes */ stateChanged: (state: DH_STATES) => any; /** Emitted when an error occurs that was not thrown intentionally */ @@ -238,9 +240,10 @@ export class DownloaderHelper extends EventEmitter { * Updates the options, can be use on pause/resume events * * @param {Object} [options={}] + * @param {String} [url=''] * @memberof DownloaderHelper */ - updateOptions(options?: object): void; + updateOptions(options?: object, url?: string): void; getOptions(): object; getMetadata(): object | null;