How to use the request.default.defaults function in request

To help you get started, we’ve selected a few request examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github openaq / openaq-fetch / adapters / au_vic.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import parallelLimit from 'async/parallelLimit';
import Bottleneck from 'bottleneck';
import { unifyMeasurementUnits } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

// Default rate limiting on API is set to 5 requests/sec.
// > Please send an email to Tools.support@epa.vic.gov.au with subject
// > 'EPA API Access Request: Increase rate-limiting' and a justified
// > reason if you want to get it increased for your subscriptions.
var maxRequestsPerSecond = 5;
var limiter = new Bottleneck({
  reservoir: maxRequestsPerSecond, // allow 5 requests
  reservoirRefreshAmount: maxRequestsPerSecond,
  reservoirRefreshInterval: 1000, // every 1000ms
  maxConcurrent: 1,
  minTime: (1000 / maxRequestsPerSecond) + 50 // to stagger requests out throught each second adding a 50ms buffer
});

export const name = 'victoria';
github openaq / openaq-fetch / adapters / buenos-aires.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import { parallel } from 'async';
import { flattenDeep } from 'lodash';
import { acceptableParameters, convertUnits } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

export const name = 'buenos-aires';
const timezone = 'America/Argentina/Buenos_Aires';

export function fetchData (source, callback) {
  request.get(source.url, (err, res, body) => {
    if (err || res.statusCode !== 200) {
      return callback({message: 'Failed to load entry point url'}, null);
    }

    let tasks = [];
    let $ = cheerio.load(body);

    const stations = $('#estacion option').filter(function (i, el) {
      // skip non working station
      return $(this).text() !== 'PALERMO';
github openaq / openaq-fetch / adapters / sweden.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import { acceptableParameters } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

exports.name = 'sweden';

exports.fetchData = function (source, cb) {
  request(source.url, function (err, res, body) {
    if (err || res.statusCode !== 200) {
      return cb({message: 'Failure to load data url.'});
    }

    // Wrap everything in a try/catch in case something goes wrong
    try {
      // Format the data
      var data = formatData(body);

      // Make sure the data is valid
      if (data === undefined) {
github openaq / openaq-fetch / adapters / beijing.js View on Github external
/**
 * This code is responsible for implementing all methods related to fetching
 * and returning data from StateAir.net data source.
 */
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { cloneDeep } from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

export const name = 'beijing';

/**
 * Fetches the data for a given source and returns an appropriate object
 * @param {object} source A valid source object
 * @param {function} cb A callback of the form cb(err, data)
 */
export const fetchData = function (source, cb) {
  request(source.url, function (err, res, body) {
    if (err || res.statusCode !== 200) {
      return cb({message: 'Failure to load data url.'});
    }

    // Wrap everything in a try/catch in case something goes wrong
    try {
github openaq / openaq-fetch / adapters / norway.js View on Github external
/**
 * This code is responsible for implementing all methods related to fetching
 * and returning data for the Norwegian data sources.
 */
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { default as moment } from 'moment-timezone';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

exports.name = 'norway';

/**
 * Fetches the data for a given source and returns an appropriate object
 * @param {object} source A valid source object
 * @param {function} cb A callback of the form cb(err, data)
 */
exports.fetchData = function (source, cb) {
  request(source.url, function (err, res, body) {
    if (err || res.statusCode !== 200) {
      return cb({message: 'Failure to load data url.'});
    }

    // Wrap everything in a try/catch in case something goes wrong
    try {
github openaq / openaq-fetch / adapters / data_gov_in.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { default as moment } from 'moment-timezone';
import { convertUnits, safeParse, acceptableParameters } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

export const name = 'data_gov_in';

export function fetchData (source, cb) {
  // Load initial results set and grab more if needed recursively
  let offset = 0;
  const limit = 1000;
  const url = `${source.url}?api-key=${process.env.DATA_GOV_IN_TOKEN}&limit=${limit}&fields=city,station,last_update,pollutant_id,pollutant_avg&format=json`;
  let results = [];

  const getResults = function (url) {
    return getData(url, (err, body) => {
      if (err) {
        return cb({message: err.message});
      }
github openaq / openaq-fetch / adapters / cpcb.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import cheerio from 'cheerio';
import { default as moment } from 'moment-timezone';
import { filter, flattenDeep, isFinite } from 'lodash';
import { parallel } from 'async';
import { acceptableParameters, convertUnits } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

export const name = 'cpcb';

export function fetchData (source, cb) {
  // Load initial page to get individual states
  request(source.url, (err, res, body) => {
    if (err || res.statusCode !== 200) {
      return cb({message: 'Failure to load data url.'});
    }

    const $ = cheerio.load(body);
    // Loop over each state and add tasks
    let tasks = [];
    const states = grabActive($);
    states.forEach((e, i) => {
      tasks.push(handleState($(e)));
github openaq / openaq-fetch / adapters / au_sa.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import { cloneDeep } from 'lodash';
import { default as moment } from 'moment-timezone';
// note: this is the 'synchronous' version (lost hours to this!)
import { default as parse } from 'csv-parse/lib/sync';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

exports.name = 'au_sa';

exports.fetchData = function (source, cb) {
  // Fetch the data
  request(source.url, function (err, res, body) {
    if (err || res.statusCode !== 200) {
      return cb({message: 'Failure to load data url.'});
    }
    // Wrap everything in a try/catch in case something goes wrong
    try {
      // Format the data
      var data = formatData(body, source);
      if (data === undefined) {
        return cb({message: 'Failure to parse data.'});
      }
github openaq / openaq-fetch / adapters / netherlands.js View on Github external
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import async from 'async';
import { removeUnwantedParameters } from '../lib/utils';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});

exports.name = 'netherlands';

exports.fetchData = function (source, cb) {
  var finalURL = source.url;
  request(finalURL, function (err, res, body) {
    if (err || res.statusCode !== 200) {
      return cb({message: 'Failure to load data url.'});
    }

    // Fetch list with available files from server
    var fileList = listApachetree(body, finalURL);

    // Filter list so it only contains the most recent group of files.
    var recentDate = _.last(fileList).date.getTime();
    var recentFiles = _.filter(fileList, function (f) {
github openaq / openaq-fetch / adapters / india.js View on Github external
/**
 * This code is responsible for implementing all methods related to
 * fetching and returning data for the Indian data sources.
 */
'use strict';

import { REQUEST_TIMEOUT } from '../lib/constants';
import { default as baseRequest } from 'request';
const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT});
import _ from 'lodash';
import { default as moment } from 'moment-timezone';
import cheerio from 'cheerio';
import log from '../lib/logger';
import { removeUnwantedParameters, convertUnits } from '../lib/utils';

exports.name = 'india';

/**
 * Fetches the data for a given source and returns an appropriate object
 * @param {object} source A valid source object
 * @param {function} cb A callback of the form cb(err, data)
 */
exports.fetchData = function (source, cb) {
  request(source.url, function (err, res, body) {
    if (err || res.statusCode !== 200) {