fix: do not accept single items for ipfs.add (#3900)

The types allow passing single items to `ipfs.addAll` and multiple items
to `ipfs.add`.

Instead, only accept single items to `ipfs.add` and streams of item to
`ipfs.addAll` and fail with a more helpful error message if you do not
do this.

BREAKING CHANGE: errors will now be thrown if multiple items are passed to `ipfs.add` or single items to `ipfs.addAll` (n.b. you can still pass a list of a single item to `ipfs.addAll`)

Author: achingbrain

packages/interface-ipfs-core/src/add-all.js

@@ -301,6 +301,26 @@ export function testAddAll (factory, options) {
       await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejected()
     })
 
+    it('should fail when passed single file objects', async () => {
+      const nonValid = { content: 'hello world' }
+
+      // @ts-expect-error nonValid is non valid
+      await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/)
+    })
+
+    it('should fail when passed single strings', async () => {
+      const nonValid = 'hello world'
+
+      await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/)
+    })
+
+    it('should fail when passed single buffers', async () => {
+      const nonValid = uint8ArrayFromString('hello world')
+
+      // @ts-expect-error nonValid is non valid
+      await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejectedWith(/single item passed/)
+    })
+
     it('should wrap content in a directory', async () => {
       const data = { path: 'testfile.txt', content: fixtures.smallFile.data }
 

packages/interface-ipfs-core/src/add.js

@@ -244,6 +244,13 @@ export function testAdd (factory, options) {
       await expect(ipfs.add(null)).to.eventually.be.rejected()
     })
 
+    it('should fail when passed multiple file objects', async () => {
+      const nonValid = [{ content: 'hello' }, { content: 'world' }]
+
+      // @ts-expect-error nonValid is non valid
+      await expect(ipfs.add(nonValid)).to.eventually.be.rejectedWith(/multiple items passed/)
+    })
+
     it('should wrap content in a directory', async () => {
       const data = { path: 'testfile.txt', content: fixtures.smallFile.data }
 

packages/ipfs-cli/src/parser.js

@@ -1,4 +1,3 @@
-
 import yargs from 'yargs'
 import { ipfsPathHelp, disablePrinting } from './utils.js'
 import { commandList } from './commands/index.js'

packages/ipfs-core-utils/package.json

@@ -38,11 +38,17 @@
     ".": {
       "import": "./src/index.js"
     },
-    "./files/normalise-input": {
-      "import": "./src/files/normalise-input.js"
+    "./files/normalise-input-single": {
+      "import": "./src/files/normalise-input-single.js"
     },
-    "./files/normalise-input.browser": {
-      "import": "./src/files/normalise-input.browser.js"
+    "./files/normalise-input-single.browser": {
+      "import": "./src/files/normalise-input-single.browser.js"
+    },
+    "./files/normalise-input-multiple": {
+      "import": "./src/files/normalise-input-multiple.js"
+    },
+    "./files/normalise-input-multiple.browser": {
+      "import": "./src/files/normalise-input-multiple.browser.js"
     },
     "./files/normalise-content": {
       "import": "./src/files/normalise-content.js"

packages/ipfs-core-utils/src/files/normalise.js renamed to packages/ipfs-core-utils/src/files/normalise-candidate-multiple.js

@@ -14,33 +14,25 @@ import {
 } from 'ipfs-unixfs'
 
 /**
+ * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate
  * @typedef {import('ipfs-core-types/src/utils').ToContent} ToContent
  * @typedef {import('ipfs-unixfs-importer').ImportCandidate} ImporterImportCandidate
- * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate
  * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream
  */
 
 /**
- * @param {ImportCandidate | ImportCandidateStream} input
+ * @param {ImportCandidateStream} input
  * @param {(content:ToContent) => Promise<AsyncIterable<Uint8Array>>} normaliseContent
  */
 // eslint-disable-next-line complexity
-export async function * normalise (input, normaliseContent) {
-  if (input === null || input === undefined) {
-    throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT')
-  }
-
+export async function * normaliseCandidateMultiple (input, normaliseContent) {
   // String
-  if (typeof input === 'string' || input instanceof String) {
-    yield toFileObject(input.toString(), normaliseContent)
-    return
-  }
-
   // Uint8Array|ArrayBuffer|TypedArray
   // Blob|File
-  if (isBytes(input) || isBlob(input)) {
-    yield toFileObject(input, normaliseContent)
-    return
+  // fs.ReadStream
+  // @ts-expect-error _readableState is a property of a node fs.ReadStream
+  if (typeof input === 'string' || input instanceof String || isBytes(input) || isBlob(input) || input._readableState) {
+    throw errCode(new Error('Unexpected input: single item passed - if you are using ipfs.allAll, please use ipfs.add instead'), 'ERR_UNEXPECTED_INPUT')
   }
 
   // Browser ReadableStream
@@ -67,42 +59,37 @@ export async function * normalise (input, normaliseContent) {
 
     // (Async)Iterable<Number>
     // (Async)Iterable<Bytes>
-    if (Number.isInteger(value) || isBytes(value)) {
-      yield toFileObject(peekable, normaliseContent)
-      return
+    if (Number.isInteger(value)) {
+      throw errCode(new Error('Unexpected input: single item passed - if you are using ipfs.allAll, please use ipfs.add instead'), 'ERR_UNEXPECTED_INPUT')
     }
 
-    // fs.ReadStream<Bytes>
+    // (Async)Iterable<fs.ReadStream>
     if (value._readableState) {
-      // @ts-ignore Node readable streams have a `.path` property so we need to pass it as the content
+      // @ts-ignore Node fs.ReadStreams have a `.path` property so we need to pass it as the content
       yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject({ content: value }, normaliseContent))
       return
     }
 
-    // (Async)Iterable<Blob>
-    // (Async)Iterable<String>
-    // (Async)Iterable<{ path, content }>
-    if (isFileObject(value) || isBlob(value) || typeof value === 'string' || value instanceof String) {
-      yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent))
+    if (isBytes(value)) {
+      yield toFileObject({ content: peekable }, normaliseContent)
       return
     }
 
     // (Async)Iterable<(Async)Iterable<?>>
     // (Async)Iterable<ReadableStream<?>>
     // ReadableStream<(Async)Iterable<?>>
     // ReadableStream<ReadableStream<?>>
-    if (value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value)) {
+    if (isFileObject(value) || value[Symbol.iterator] || value[Symbol.asyncIterator] || isReadableStream(value) || isBlob(value)) {
       yield * map(peekable, (/** @type {ImportCandidate} */ value) => toFileObject(value, normaliseContent))
       return
     }
   }
 
   // { path, content: ? }
-  // Note: Detected _after_ (Async)Iterable<?> because Node.js streams have a
+  // Note: Detected _after_ (Async)Iterable<?> because Node.js fs.ReadStreams have a
   // `path` property that passes this check.
   if (isFileObject(input)) {
-    yield toFileObject(input, normaliseContent)
-    return
+    throw errCode(new Error('Unexpected input: single item passed - if you are using ipfs.allAll, please use ipfs.add instead'), 'ERR_UNEXPECTED_INPUT')
   }
 
   throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')

packages/ipfs-core-utils/src/files/normalise-candidate-single.js

@@ -0,0 +1,111 @@
+import errCode from 'err-code'
+import browserStreamToIt from 'browser-readablestream-to-it'
+import itPeekable from 'it-peekable'
+import {
+  isBytes,
+  isBlob,
+  isReadableStream,
+  isFileObject
+} from './utils.js'
+import {
+  parseMtime,
+  parseMode
+} from 'ipfs-unixfs'
+
+/**
+ * @typedef {import('ipfs-core-types/src/utils').ToContent} ToContent
+ * @typedef {import('ipfs-unixfs-importer').ImportCandidate} ImporterImportCandidate
+ * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate
+ * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream
+ */
+
+/**
+ * @param {ImportCandidate} input
+ * @param {(content:ToContent) => Promise<AsyncIterable<Uint8Array>>} normaliseContent
+ */
+// eslint-disable-next-line complexity
+export async function * normaliseCandidateSingle (input, normaliseContent) {
+  if (input === null || input === undefined) {
+    throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT')
+  }
+
+  // String
+  if (typeof input === 'string' || input instanceof String) {
+    yield toFileObject(input.toString(), normaliseContent)
+    return
+  }
+
+  // Uint8Array|ArrayBuffer|TypedArray
+  // Blob|File
+  if (isBytes(input) || isBlob(input)) {
+    yield toFileObject(input, normaliseContent)
+    return
+  }
+
+  // Browser ReadableStream
+  if (isReadableStream(input)) {
+    input = browserStreamToIt(input)
+  }
+
+  // Iterable<?>
+  if (Symbol.iterator in input || Symbol.asyncIterator in input) {
+    // @ts-ignore it's (async)iterable
+    const peekable = itPeekable(input)
+
+    /** @type {any} value **/
+    const { value, done } = await peekable.peek()
+
+    if (done) {
+      // make sure empty iterators result in empty files
+      yield { content: [] }
+      return
+    }
+
+    peekable.push(value)
+
+    // (Async)Iterable<Number>
+    // (Async)Iterable<Bytes>
+    // (Async)Iterable<String>
+    if (Number.isInteger(value) || isBytes(value) || typeof value === 'string' || value instanceof String) {
+      yield toFileObject(peekable, normaliseContent)
+      return
+    }
+
+    throw errCode(new Error('Unexpected input: multiple items passed - if you are using ipfs.add, please use ipfs.addAll instead'), 'ERR_UNEXPECTED_INPUT')
+  }
+
+  // { path, content: ? }
+  // Note: Detected _after_ (Async)Iterable<?> because Node.js fs.ReadStreams have a
+  // `path` property that passes this check.
+  if (isFileObject(input)) {
+    yield toFileObject(input, normaliseContent)
+    return
+  }
+
+  throw errCode(new Error('Unexpected input: cannot convert "' + typeof input + '" into ImportCandidate'), 'ERR_UNEXPECTED_INPUT')
+}
+
+/**
+ * @param {ImportCandidate} input
+ * @param {(content:ToContent) => Promise<AsyncIterable<Uint8Array>>} normaliseContent
+ */
+async function toFileObject (input, normaliseContent) {
+  // @ts-ignore - Those properties don't exist on most input types
+  const { path, mode, mtime, content } = input
+
+  /** @type {ImporterImportCandidate} */
+  const file = {
+    path: path || '',
+    mode: parseMode(mode),
+    mtime: parseMtime(mtime)
+  }
+
+  if (content) {
+    file.content = await normaliseContent(content)
+  } else if (!path) { // Not already a file object with path or content prop
+    // @ts-ignore - input still can be different ToContent
+    file.content = await normaliseContent(input)
+  }
+
+  return file
+}

packages/ipfs-core-utils/src/files/normalise-content.browser.js

@@ -9,7 +9,7 @@ import {
 } from './utils.js'
 
 /**
- * @param {import('./normalise').ToContent} input
+ * @param {import('ipfs-core-types/src/utils').ToContent} input
  */
 export async function normaliseContent (input) {
   // Bytes

packages/ipfs-core-utils/src/files/normalise-content.js

@@ -12,31 +12,29 @@ import {
 } from './utils.js'
 
 /**
- * @param {import('./normalise').ToContent} input
+ * @template T
+ * @param {T} thing
  */
-export async function normaliseContent (input) {
-  return toAsyncGenerator(input)
+async function * toAsyncIterable (thing) {
+  yield thing
 }
 
 /**
- * @param {import('./normalise').ToContent} input
+ * @param {import('ipfs-core-types/src/utils').ToContent} input
  */
-async function * toAsyncGenerator (input) {
+export async function normaliseContent (input) {
   // Bytes | String
   if (isBytes(input)) {
-    yield toBytes(input)
-    return
+    return toAsyncIterable(toBytes(input))
   }
 
   if (typeof input === 'string' || input instanceof String) {
-    yield toBytes(input.toString())
-    return
+    return toAsyncIterable(toBytes(input.toString()))
   }
 
   // Blob
   if (isBlob(input)) {
-    yield * blobToIt(input)
-    return
+    return blobToIt(input)
   }
 
   // Browser stream
@@ -54,22 +52,19 @@ async function * toAsyncGenerator (input) {
 
     if (done) {
       // make sure empty iterators result in empty files
-      yield * []
-      return
+      return toAsyncIterable(new Uint8Array(0))
     }
 
     peekable.push(value)
 
     // (Async)Iterable<Number>
     if (Number.isInteger(value)) {
-      yield Uint8Array.from((await all(peekable)))
-      return
+      return toAsyncIterable(Uint8Array.from(await all(peekable)))
     }
 
     // (Async)Iterable<Bytes|String>
     if (isBytes(value) || typeof value === 'string' || value instanceof String) {
-      yield * map(peekable, toBytes)
-      return
+      return map(peekable, toBytes)
     }
   }
 

packages/ipfs-core-utils/src/files/normalise-input.browser.js renamed to packages/ipfs-core-utils/src/files/normalise-input-multiple.browser.js

@@ -1,25 +1,24 @@
 import { normaliseContent } from './normalise-content.browser.js'
-import { normalise } from './normalise.js'
+import { normaliseCandidateMultiple } from './normalise-candidate-multiple.js'
 
 /**
  * @typedef {import('ipfs-core-types/src/utils').ImportCandidateStream} ImportCandidateStream
- * @typedef {import('ipfs-core-types/src/utils').ImportCandidate} ImportCandidate
  * @typedef {import('ipfs-core-types/src/utils').BrowserImportCandidate} BrowserImportCandidate
  */
 
 /**
- * Transforms any of the `ipfs.add` input types into
+ * Transforms any of the `ipfs.addAll` input types into
  *
  * ```
  * AsyncIterable<{ path, mode, mtime, content: Blob }>
  * ```
  *
  * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options
  *
- * @param {ImportCandidate | ImportCandidateStream} input
+ * @param {ImportCandidateStream} input
  * @returns {AsyncGenerator<BrowserImportCandidate, void, undefined>}
  */
 export function normaliseInput (input) {
-  // @ts-ignore normaliseContent returns Blob and not AsyncIterator
-  return normalise(input, normaliseContent)
+  // @ts-expect-error browser normaliseContent returns a Blob not an AsyncIterable<Uint8Array>
+  return normaliseCandidateMultiple(input, normaliseContent, true)
 }