feat: remove ky from http-client and utils (#2810)

- we now use a simpler http lib 
- bundle size reduced
- some tests are now more stable
- search params handling is cleaner 

closes #2801

.aegir.js

@@ -14,7 +14,7 @@ let echoServer
 const webpack = require('webpack')
 
 module.exports = {
-  bundlesize: { maxSize: '94kB' },
+  bundlesize: { maxSize: '89kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main']

examples/files-api/files-api.js

@@ -1,6 +1,6 @@
 /* eslint-disable no-console */
 'use strict'
-
+const { Buffer } = require('buffer')
 // Run `ipfs daemon` in your terminal to start the IPFS daemon
 // Look for `API server listening on /ip4/127.0.0.1/tcp/5001`
 const ipfs = require('../../src')('/ip4/127.0.0.1/tcp/5001')

examples/name-api/index.js

@@ -1,5 +1,6 @@
 /* eslint-disable no-console */
 'use strict'
+const { Buffer } = require('buffer')
 const ipfsHttp = require('ipfs-http-client')
 const ipfs = ipfsHttp('/ip4/127.0.0.1/tcp/5001')
 

package.json

@@ -51,23 +51,21 @@
     "ipld-dag-cbor": "^0.15.1",
     "ipld-dag-pb": "^0.18.2",
     "ipld-raw": "^4.0.1",
+    "iso-url": "^0.4.6",
     "it-tar": "^1.2.1",
     "it-to-stream": "^0.1.1",
-    "iterable-ndjson": "^1.1.0",
-    "ky": "^0.15.0",
-    "ky-universal": "^0.3.0",
     "merge-options": "^2.0.0",
     "multiaddr": "^7.2.1",
     "multiaddr-to-uri": "^5.1.0",
     "multibase": "^0.6.0",
     "multicodec": "^1.0.0",
     "multihashes": "^0.4.14",
+    "node-fetch": "^2.6.0",
     "parse-duration": "^0.1.2",
     "stream-to-it": "^0.2.0"
   },
   "devDependencies": {
     "aegir": "^21.3.0",
-    "async": "^3.1.0",
     "browser-process-platform": "^0.1.1",
     "cross-env": "^7.0.0",
     "go-ipfs-dep": "0.4.23-3",

src/add/form-data.browser.js

@@ -10,6 +10,7 @@ exports.toFormData = async input => {
   let i = 0
 
   for await (const file of files) {
+    // TODO FormData.append doesnt have a 4th arg
     const headers = {}
 
     if (file.mtime !== undefined && file.mtime !== null) {
@@ -34,13 +35,23 @@ exports.toFormData = async input => {
         bufs.push(chunk)
       }
 
-      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path), {
-        header: headers
-      })
+      formData.append(
+        `file-${i}`,
+        new Blob(bufs, { type: 'application/octet-stream' }),
+        encodeURIComponent(file.path)
+        // {
+        //   header: headers
+        // }
+      )
     } else {
-      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path), {
-        header: headers
-      })
+      formData.append(
+        `dir-${i}`,
+        new Blob([], { type: 'application/x-directory' }),
+        encodeURIComponent(file.path)
+        // {
+        //   header: headers
+        // }
+      )
     }
 
     i++

src/add/form-data.js

@@ -4,7 +4,6 @@ const FormData = require('form-data')
 const { Buffer } = require('buffer')
 const toStream = require('it-to-stream')
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
-const { isElectronRenderer } = require('ipfs-utils/src/env')
 const mtimeToObject = require('../lib/mtime-to-object')
 
 exports.toFormData = async input => {
@@ -59,10 +58,3 @@ exports.toFormData = async input => {
 
   return formData
 }
-
-// TODO remove this when upstream fix for ky-universal is merged
-// https://github.com/sindresorhus/ky-universal/issues/9
-// also this should only be necessary when nodeIntegration is false in electron renderer
-if (isElectronRenderer) {
-  exports.toFormData = require('./form-data.browser').toFormData
-}

src/add/index.js

@@ -1,51 +1,36 @@
 'use strict'
 
-const ndjson = require('iterable-ndjson')
 const CID = require('cids')
-const configure = require('../lib/configure')
-const toIterable = require('stream-to-it/source')
+const merge = require('merge-options')
 const { toFormData } = require('./form-data')
 const toCamel = require('../lib/object-to-camel')
+const configure = require('../lib/configure')
 
-module.exports = configure(({ ky }) => {
-  return async function * add (input, options) {
-    options = options || {}
-
-    const searchParams = new URLSearchParams(options.searchParams)
-
-    searchParams.set('stream-channels', true)
-    if (options.chunker) searchParams.set('chunker', options.chunker)
-    if (options.cidVersion) searchParams.set('cid-version', options.cidVersion)
-    if (options.cidBase) searchParams.set('cid-base', options.cidBase)
-    if (options.enableShardingExperiment != null) searchParams.set('enable-sharding-experiment', options.enableShardingExperiment)
-    if (options.hashAlg) searchParams.set('hash', options.hashAlg)
-    if (options.onlyHash != null) searchParams.set('only-hash', options.onlyHash)
-    if (options.pin != null) searchParams.set('pin', options.pin)
-    if (options.progress) searchParams.set('progress', true)
-    if (options.quiet != null) searchParams.set('quiet', options.quiet)
-    if (options.quieter != null) searchParams.set('quieter', options.quieter)
-    if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves)
-    if (options.shardSplitThreshold) searchParams.set('shard-split-threshold', options.shardSplitThreshold)
-    if (options.silent) searchParams.set('silent', options.silent)
-    if (options.trickle != null) searchParams.set('trickle', options.trickle)
-    if (options.wrapWithDirectory != null) searchParams.set('wrap-with-directory', options.wrapWithDirectory)
-    if (options.preload != null) searchParams.set('preload', options.preload)
-    if (options.fileImportConcurrency != null) searchParams.set('file-import-concurrency', options.fileImportConcurrency)
-    if (options.blockWriteConcurrency != null) searchParams.set('block-write-concurrency', options.blockWriteConcurrency)
+module.exports = configure((api) => {
+  return async function * add (input, options = {}) {
+    const progressFn = options.progress
+    options = merge(
+      options,
+      {
+        'stream-channels': true,
+        progress: Boolean(progressFn),
+        hash: options.hashAlg // TODO fix this either is hash or hashAlg
+      }
+    )
 
-    const res = await ky.post('add', {
+    const res = await api.ndjson('add', {
+      method: 'POST',
+      searchParams: options,
+      body: await toFormData(input),
       timeout: options.timeout,
-      signal: options.signal,
-      headers: options.headers,
-      searchParams,
-      body: await toFormData(input)
+      signal: options.signal
     })
 
-    for await (let file of ndjson(toIterable(res.body))) {
+    for await (let file of res) {
       file = toCamel(file)
 
-      if (options.progress && file.bytes) {
-        options.progress(file.bytes)
+      if (progressFn && file.bytes) {
+        progressFn(file.bytes)
       } else {
         yield toCoreInterface(file)
       }

src/bitswap/stat.js

@@ -1,21 +1,18 @@
 'use strict'
 
-const configure = require('../lib/configure')
-const Big = require('bignumber.js')
+const { BigNumber } = require('bignumber.js')
 const CID = require('cids')
+const configure = require('../lib/configure')
 
-module.exports = configure(({ ky }) => {
-  return async (options) => {
-    options = options || {}
-
-    const res = await ky.post('bitswap/stat', {
+module.exports = configure(api => {
+  return async (options = {}) => {
+    const res = await api.post('bitswap/stat', {
+      searchParams: options,
       timeout: options.timeout,
-      signal: options.signal,
-      headers: options.headers,
-      searchParams: options.searchParams
-    }).json()
+      signal: options.signal
+    })
 
-    return toCoreInterface(res)
+    return toCoreInterface(await res.json())
   }
 })
 
@@ -24,11 +21,11 @@ function toCoreInterface (res) {
     provideBufLen: res.ProvideBufLen,
     wantlist: (res.Wantlist || []).map(k => new CID(k['/'])),
     peers: (res.Peers || []),
-    blocksReceived: new Big(res.BlocksReceived),
-    dataReceived: new Big(res.DataReceived),
-    blocksSent: new Big(res.BlocksSent),
-    dataSent: new Big(res.DataSent),
-    dupBlksReceived: new Big(res.DupBlksReceived),
-    dupDataReceived: new Big(res.DupDataReceived)
+    blocksReceived: new BigNumber(res.BlocksReceived),
+    dataReceived: new BigNumber(res.DataReceived),
+    blocksSent: new BigNumber(res.BlocksSent),
+    dataSent: new BigNumber(res.DataSent),
+    dupBlksReceived: new BigNumber(res.DupBlksReceived),
+    dupDataReceived: new BigNumber(res.DupDataReceived)
   }
 }

src/bitswap/unwant.js

@@ -3,25 +3,16 @@
 const CID = require('cids')
 const configure = require('../lib/configure')
 
-module.exports = configure(({ ky }) => {
-  return async (cid, options) => {
-    options = options || {}
+module.exports = configure(api => {
+  return async (cid, options = {}) => {
+    options.arg = typeof cid === 'string' ? cid : new CID(cid).toString()
 
-    const searchParams = new URLSearchParams(options.searchParams)
-
-    if (typeof cid === 'string') {
-      searchParams.set('arg', cid)
-    } else {
-      searchParams.set('arg', new CID(cid).toString())
-    }
-
-    const res = await ky.post('bitswap/unwant', {
+    const res = await api.post('bitswap/unwant', {
       timeout: options.timeout,
       signal: options.signal,
-      headers: options.headers,
-      searchParams
-    }).json()
+      searchParams: options
+    })
 
-    return res
+    return res.json()
   }
 })

src/bitswap/wantlist.js

@@ -3,26 +3,17 @@
 const CID = require('cids')
 const configure = require('../lib/configure')
 
-module.exports = configure(({ ky }) => {
-  return async (peerId, options) => {
-    options = options || {}
-
-    const searchParams = new URLSearchParams(options.searchParams)
-
-    if (peerId) {
-      if (typeof peerId === 'string') {
-        searchParams.set('peer', peerId)
-      } else {
-        searchParams.set('peer', new CID(peerId).toString())
-      }
+module.exports = configure(api => {
+  return async (peer, options = {}) => {
+    if (peer) {
+      options.peer = typeof peer === 'string' ? peer : new CID(peer).toString()
     }
 
-    const res = await ky.post('bitswap/wantlist', {
+    const res = await (await api.post('bitswap/wantlist', {
       timeout: options.timeout,
       signal: options.signal,
-      headers: options.headers,
-      searchParams
-    }).json()
+      searchParams: options
+    })).json()
 
     return (res.Keys || []).map(k => new CID(k['/']))
   }

src/block/get.js

@@ -5,21 +5,17 @@ const CID = require('cids')
 const { Buffer } = require('buffer')
 const configure = require('../lib/configure')
 
-module.exports = configure(({ ky }) => {
-  return async (cid, options) => {
+module.exports = configure(api => {
+  return async (cid, options = {}) => {
     cid = new CID(cid)
-    options = options || {}
+    options.arg = cid.toString()
 
-    const searchParams = new URLSearchParams(options.searchParams)
-    searchParams.set('arg', `${cid}`)
-
-    const data = await ky.post('block/get', {
+    const rsp = await api.post('block/get', {
       timeout: options.timeout,
       signal: options.signal,
-      headers: options.headers,
-      searchParams
-    }).arrayBuffer()
+      searchParams: options
+    })
 
-    return new Block(Buffer.from(data), cid)
+    return new Block(Buffer.from(await rsp.arrayBuffer()), cid)
   }
 })

src/block/put.js

@@ -3,13 +3,11 @@
 const Block = require('ipfs-block')
 const CID = require('cids')
 const multihash = require('multihashes')
-const configure = require('../lib/configure')
 const toFormData = require('../lib/buffer-to-form-data')
+const configure = require('../lib/configure')
 
-module.exports = configure(({ ky }) => {
-  async function put (data, options) {
-    options = options || {}
-
+module.exports = configure(api => {
+  async function put (data, options = {}) {
     if (Block.isBlock(data)) {
       const { name, length } = multihash.decode(data.cid.multihash)
       options = {
@@ -33,22 +31,15 @@ module.exports = configure(({ ky }) => {
       delete options.cid
     }
 
-    const searchParams = new URLSearchParams(options.searchParams)
-    if (options.format) searchParams.set('format', options.format)
-    if (options.mhtype) searchParams.set('mhtype', options.mhtype)
-    if (options.mhlen) searchParams.set('mhlen', options.mhlen)
-    if (options.pin != null) searchParams.set('pin', options.pin)
-    if (options.version != null) searchParams.set('version', options.version)
-
     let res
     try {
-      res = await ky.post('block/put', {
+      const response = await api.post('block/put', {
         timeout: options.timeout,
         signal: options.signal,
-        headers: options.headers,
-        searchParams,
+        searchParams: options,
         body: toFormData(data)
-      }).json()
+      })
+      res = await response.json()
     } catch (err) {
       // Retry with "protobuf"/"cbor" format for go-ipfs
       // TODO: remove when https://github.com/ipfs/go-cid/issues/75 resolved