How to use the apache-arrow/Arrow.es5.min.RecordBatchReader.readAll function in apache-arrow

To help you get started, we’ve selected a few apache-arrow examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github uber-web / loaders.gl / modules / arrow / src / lib / parse-arrow-in-batches.js View on Github external
export async function parseArrowInBatches(asyncIterator, options) {
  // Creates the appropriate RecordBatchReader subclasses from the input
  // This will also close the underlying source in case of early termination or errors
  const readers = await RecordBatchReader.readAll(asyncIterator);

  // As an optimization, return a non-async iterator
  if (isIterable(readers)) {
    return (function* arrowIterator() {
      for (const reader of readers) {
        for (const batch of reader) {
          yield processBatch(batch, reader);
        }
        break; // only processing one stream of batches
      }
    })();
  }

  return (async function* arrowAsyncIterator() {
    for await (const reader of readers) {
      for await (const batch of reader) {
github uber-web / loaders.gl / modules / arrow / src / lib / parse-arrow-in-batches.js View on Github external
export async function parseArrowInBatchesSync(iterator, options) {
  // Creates the appropriate RecordBatchReader subclasses from the input
  // This will also close the underlying source in case of early termination or errors
  const readers = RecordBatchReader.readAll(iterator);

  // Check that `readers` is not a Promise, and is iterable
  if (isIterable(readers) || isIterator(readers)) {
    return (function* arrowIterator() {
      for (const reader of readers) {
        for (const batch of reader) {
          yield processBatch(batch);
        }
        break; // only processing one stream of batches
      }
    })();
  }

  return assert(false);
}