Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
function fetchPollutants (source, stations) {
const pollutants = acceptableParameters.map(
(pollutant) => pollutant === 'pm25' ? 'PM2.5' : pollutant.toUpperCase()
);
return new MultiStream(
pollutants.map(pollutant => {
const url = source.url + source.country + '_' + pollutant + '.csv';
const timeLastInsert = moment().utc().subtract(2, 'hours');
let header;
return new StringStream()
.use(stream => {
const resp = request.get({url})
.on('response', ({statusCode}) => {
+statusCode !== 200
? stream.end()
: resp.pipe(stream);
});
return stream;
})
.CSVParse({header: false, delimiter: ',', skipEmptyLines: true})
let $;
try {
$ = cheerio.load(response.body);
} catch (e) {
throw new FetchError(SOURCE_URL_CANNOT_PARSE_DATA, source, e);
}
const provinces = $('#provincia option')
.filter(function (i, el) {
return Number($(this).attr('value')) >= 0;
})
.map(function (i, el) {
return { id: $(this).attr('value'), name: $(this).text() };
}).get();
const out = new MultiStream();
provinces.forEach(
async province => {
const provinceHourlyURL = `${baseUrl}${provinceQueryPath}?provincia=${province.id}&dati=${hourlyAvgParam}`;
const provinceDailyURL = `${baseUrl}${provinceQueryPath}?provincia=${province.id}&dati=${dailyAvgParam}`;
out.add(await handleProvince(province.name, provinceHourlyURL, hourlyAvgPeriod, source));
out.add(await handleProvince(province.name, provinceDailyURL, dailyAvgPeriod, source));
}
);
return out.mux();
};
if (response.statusCode !== 200) {
throw new FetchError(SOURCE_URL_CANNOT_PARSE_DATA, source, null);
}
log.verbose(`Loading province information from ${url}`);
const $ = cheerio.load(response.body);
const pollutantURLs = $('a').map(function () {
const pollutant = $(this).text().toLowerCase().replace('.', '');
const currentParameters = getParameters(averagingPeriod);
if (currentParameters.indexOf(pollutant) >= 0) {
const href = $(this).attr('href');
return `${baseUrl}${href}`;
}
}).get();
return new MultiStream(
await Promise.all(
pollutantURLs.map(dataUrl => getStream(name, dataUrl, averagingPeriod, source, url))
)
).mux();
};