How to use the scramjet.MultiStream function in scramjet

To help you get started, we’ve selected a few scramjet examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github openaq / openaq-fetch / adapters / eea-direct.js View on Github external
function fetchPollutants (source, stations) {
  const pollutants = acceptableParameters.map(
    (pollutant) => pollutant === 'pm25' ? 'PM2.5' : pollutant.toUpperCase()
  );

  return new MultiStream(
    pollutants.map(pollutant => {
      const url = source.url + source.country + '_' + pollutant + '.csv';
      const timeLastInsert = moment().utc().subtract(2, 'hours');
      let header;

      return new StringStream()
        .use(stream => {
          const resp = request.get({url})
            .on('response', ({statusCode}) => {
              +statusCode !== 200
                ? stream.end()
                : resp.pipe(stream);
            });
          return stream;
        })
        .CSVParse({header: false, delimiter: ',', skipEmptyLines: true})
github openaq / openaq-fetch / adapters / arpalazio.js View on Github external
let $;
  try {
    $ = cheerio.load(response.body);
  } catch (e) {
    throw new FetchError(SOURCE_URL_CANNOT_PARSE_DATA, source, e);
  }

  const provinces = $('#provincia option')
    .filter(function (i, el) {
      return Number($(this).attr('value')) >= 0;
    })
    .map(function (i, el) {
      return { id: $(this).attr('value'), name: $(this).text() };
    }).get();

  const out = new MultiStream();
  provinces.forEach(
    async province => {
      const provinceHourlyURL = `${baseUrl}${provinceQueryPath}?provincia=${province.id}&dati=${hourlyAvgParam}`;
      const provinceDailyURL = `${baseUrl}${provinceQueryPath}?provincia=${province.id}&dati=${dailyAvgParam}`;

      out.add(await handleProvince(province.name, provinceHourlyURL, hourlyAvgPeriod, source));
      out.add(await handleProvince(province.name, provinceDailyURL, dailyAvgPeriod, source));
    }
  );

  return out.mux();
};
github openaq / openaq-fetch / adapters / arpalazio.js View on Github external
if (response.statusCode !== 200) {
    throw new FetchError(SOURCE_URL_CANNOT_PARSE_DATA, source, null);
  }
  log.verbose(`Loading province information from ${url}`);

  const $ = cheerio.load(response.body);
  const pollutantURLs = $('a').map(function () {
    const pollutant = $(this).text().toLowerCase().replace('.', '');
    const currentParameters = getParameters(averagingPeriod);
    if (currentParameters.indexOf(pollutant) >= 0) {
      const href = $(this).attr('href');
      return `${baseUrl}${href}`;
    }
  }).get();

  return new MultiStream(
    await Promise.all(
      pollutantURLs.map(dataUrl => getStream(name, dataUrl, averagingPeriod, source, url))
    )
  ).mux();
};

scramjet

Lightweight and real-time data functional stream programming framework like event-stream, written in ES6 using async await with multi-threading and typescript support

MIT
Latest version published 8 months ago

Package Health Score

62 / 100
Full package analysis