Skip to content

Commit 25cae2e

Browse files
authoredMar 24, 2022
feat: implement local dns cache (#132)
1 parent 667df07 commit 25cae2e

File tree

7 files changed

+413
-26
lines changed

7 files changed

+413
-26
lines changed
 

‎README.md

+12
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ pooling, proxies, retries, [and more](#features)!
2929
* [`opts.retry`](#opts-retry)
3030
* [`opts.onRetry`](#opts-onretry)
3131
* [`opts.integrity`](#opts-integrity)
32+
* [`opts.dns`](#opts-dns)
3233
* [Message From Our Sponsors](#wow)
3334

3435
### Example
@@ -67,6 +68,7 @@ fetch('https://registry.npmjs.org/make-fetch-happen').then(res => {
6768
* Transparent gzip and deflate support
6869
* [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) support
6970
* Literally punches nazis
71+
* Built in DNS cache
7072
* (PENDING) Range request caching and resuming
7173

7274
### Contributing
@@ -146,6 +148,7 @@ make-fetch-happen augments the `minipass-fetch` API with additional features ava
146148
* [`opts.retry`](#opts-retry) - Request retry settings
147149
* [`opts.onRetry`](#opts-onretry) - a function called whenever a retry is attempted
148150
* [`opts.integrity`](#opts-integrity) - [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) metadata.
151+
* [`opts.dns`](#opts-dns) - DNS cache options
149152

150153
#### <a name="opts-cache-path"></a> `> opts.cachePath`
151154

@@ -387,3 +390,12 @@ fetch('https://malicious-registry.org/make-fetch-happen/-/make-fetch-happen-1.0.
387390
integrity: 'sha1-o47j7zAYnedYFn1dF/fR9OV3z8Q='
388391
}) // Error: EINTEGRITY
389392
```
393+
394+
#### <a name="opts-dns"></a> `> opts.dns`
395+
396+
An object that provides options for the built-in DNS cache. The following options are available:
397+
398+
Note: Due to limitations in the current proxy agent implementation, users of proxies will not benefit from the DNS cache.
399+
400+
* `ttl`: Milliseconds to keep cached DNS responses for. Defaults to `5 * 60 * 1000` (5 minutes)
401+
* `lookup`: A custom lookup function, see [`dns.lookup()`](https://nodejs.org/api/dns.html#dnslookuphostname-options-callback) for implementation details. Defaults to `require('dns').lookup`.

‎lib/agent.js

+5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
const LRU = require('lru-cache')
33
const url = require('url')
44
const isLambda = require('is-lambda')
5+
const dns = require('./dns.js')
56

67
const AGENT_CACHE = new LRU({ max: 50 })
78
const HttpAgent = require('agentkeepalive')
@@ -77,11 +78,13 @@ function getAgent (uri, opts) {
7778
rejectUnauthorized: opts.rejectUnauthorized,
7879
timeout: agentTimeout,
7980
freeSocketTimeout: 15000,
81+
lookup: dns.getLookup(opts.dns),
8082
}) : new HttpAgent({
8183
maxSockets: agentMaxSockets,
8284
localAddress: opts.localAddress,
8385
timeout: agentTimeout,
8486
freeSocketTimeout: 15000,
87+
lookup: dns.getLookup(opts.dns),
8588
})
8689
AGENT_CACHE.set(key, agent)
8790
return agent
@@ -171,6 +174,8 @@ const HttpsProxyAgent = require('https-proxy-agent')
171174
const SocksProxyAgent = require('socks-proxy-agent')
172175
module.exports.getProxy = getProxy
173176
function getProxy (proxyUrl, opts, isHttps) {
177+
// our current proxy agents do not support an overridden dns lookup method, so will not
178+
// benefit from the dns cache
174179
const popts = {
175180
host: proxyUrl.hostname,
176181
port: proxyUrl.port,

‎lib/dns.js

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
const LRUCache = require('lru-cache')
2+
const dns = require('dns')
3+
4+
const defaultOptions = exports.defaultOptions = {
5+
family: undefined,
6+
hints: dns.ADDRCONFIG,
7+
all: false,
8+
verbatim: true,
9+
}
10+
11+
const lookupCache = exports.lookupCache = new LRUCache({ max: 50 })
12+
13+
// this is a factory so that each request can have its own opts (i.e. ttl)
14+
// while still sharing the cache across all requests
15+
exports.getLookup = (dnsOptions) => {
16+
return (hostname, options, callback) => {
17+
if (typeof options === 'function') {
18+
callback = options
19+
options = null
20+
} else if (typeof options === 'number') {
21+
options = { family: options }
22+
}
23+
24+
options = { ...defaultOptions, ...options }
25+
26+
const key = JSON.stringify({
27+
hostname,
28+
family: options.family,
29+
hints: options.hints,
30+
all: options.all,
31+
verbatim: options.verbatim,
32+
})
33+
34+
if (lookupCache.has(key)) {
35+
const [address, family] = lookupCache.get(key)
36+
process.nextTick(callback, null, address, family)
37+
return
38+
}
39+
40+
dnsOptions.lookup(hostname, options, (err, address, family) => {
41+
if (err) {
42+
return callback(err)
43+
}
44+
45+
lookupCache.set(key, [address, family], { ttl: dnsOptions.ttl })
46+
return callback(null, address, family)
47+
})
48+
}
49+
}

‎lib/options.js

+4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
const dns = require('dns')
2+
13
const conditionalHeaders = [
24
'if-modified-since',
35
'if-none-match',
@@ -26,6 +28,8 @@ const configureOptions = (opts) => {
2628
options.retry = { retries: 0, ...options.retry }
2729
}
2830

31+
options.dns = { ttl: 5 * 60 * 1000, lookup: dns.lookup, ...options.dns }
32+
2933
options.cache = options.cache || 'default'
3034
if (options.cache === 'default') {
3135
const hasConditionalHeader = Object.keys(options.headers || {}).some((name) => {

‎test/agent.js

+5-5
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ t.test('agent: false returns false', async t => {
4747
})
4848

4949
t.test('all expected options passed down to HttpAgent', async t => {
50-
t.same(agent('http://foo.com/bar', OPTS), {
50+
t.match(agent('http://foo.com/bar', OPTS), {
5151
__type: 'http',
5252
maxSockets: 5,
5353
localAddress: 'localAddress',
@@ -57,7 +57,7 @@ t.test('all expected options passed down to HttpAgent', async t => {
5757
})
5858

5959
t.test('timeout 0 keeps timeout 0', async t => {
60-
t.same(agent('http://foo.com/bar', { ...OPTS, timeout: 0 }), {
60+
t.match(agent('http://foo.com/bar', { ...OPTS, timeout: 0 }), {
6161
__type: 'http',
6262
maxSockets: 5,
6363
localAddress: 'localAddress',
@@ -67,7 +67,7 @@ t.test('timeout 0 keeps timeout 0', async t => {
6767
})
6868

6969
t.test('no max sockets gets 15 max sockets', async t => {
70-
t.same(agent('http://foo.com/bar', { ...OPTS, maxSockets: undefined }), {
70+
t.match(agent('http://foo.com/bar', { ...OPTS, maxSockets: undefined }), {
7171
__type: 'http',
7272
maxSockets: 15,
7373
localAddress: 'localAddress',
@@ -77,7 +77,7 @@ t.test('no max sockets gets 15 max sockets', async t => {
7777
})
7878

7979
t.test('no timeout gets timeout 0', async t => {
80-
t.same(agent('http://foo.com/bar', { ...OPTS, timeout: undefined }), {
80+
t.match(agent('http://foo.com/bar', { ...OPTS, timeout: undefined }), {
8181
__type: 'http',
8282
maxSockets: 5,
8383
localAddress: 'localAddress',
@@ -87,7 +87,7 @@ t.test('no timeout gets timeout 0', async t => {
8787
})
8888

8989
t.test('all expected options passed down to HttpsAgent', async t => {
90-
t.same(agent('https://foo.com/bar', OPTS), {
90+
t.match(agent('https://foo.com/bar', OPTS), {
9191
__type: 'https',
9292
ca: 'ca',
9393
cert: 'cert',

‎test/dns.js

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
const t = require('tap')
2+
3+
const dns = require('../lib/dns.js')
4+
const DEFAULT_OPTS = { ttl: 5 * 60 * 1000 }
5+
6+
t.afterEach(() => dns.lookupCache.clear())
7+
8+
t.test('supports no options passed', async (t) => {
9+
let lookupCalled = 0
10+
const fakeLookup = (hostname, options, callback) => {
11+
lookupCalled += 1
12+
t.match(options, dns.defaultOptions, 'applied default options')
13+
process.nextTick(callback, null, '127.0.0.1', 4)
14+
}
15+
const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
16+
17+
return new Promise((resolve) => {
18+
lookup('localhost', (err, address, family) => {
19+
t.equal(err, null, 'no error')
20+
t.equal(address, '127.0.0.1', 'got address')
21+
t.equal(family, 4, 'got family')
22+
t.equal(lookupCalled, 1, 'lookup was called once')
23+
resolve()
24+
})
25+
})
26+
})
27+
28+
t.test('supports family passed directly as options', async (t) => {
29+
let lookupCalled = 0
30+
const fakeLookup = (hostname, options, callback) => {
31+
lookupCalled += 1
32+
t.match(options, { ...dns.defaultOptions, family: 4 }, 'kept family setting')
33+
process.nextTick(callback, null, '127.0.0.1', 4)
34+
}
35+
const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
36+
37+
return new Promise((resolve) => {
38+
lookup('localhost', 4, (err, address, family) => {
39+
t.equal(err, null, 'no error')
40+
t.equal(address, '127.0.0.1', 'got address')
41+
t.equal(family, 4, 'got family')
42+
t.equal(lookupCalled, 1, 'lookup was called once')
43+
resolve()
44+
})
45+
})
46+
})
47+
48+
t.test('reads from cache', async (t) => {
49+
let lookupCalled = 0
50+
const fakeLookup = (hostname, options, callback) => {
51+
lookupCalled += 1
52+
t.match(options, dns.defaultOptions, 'applied default options')
53+
process.nextTick(callback, null, '127.0.0.1', 4)
54+
}
55+
const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
56+
57+
return new Promise((resolve) => {
58+
lookup('localhost', (err, address, family) => {
59+
t.equal(err, null, 'no error')
60+
t.equal(address, '127.0.0.1', 'got address')
61+
t.equal(family, 4, 'got family')
62+
t.equal(lookupCalled, 1, 'lookup was called once')
63+
resolve()
64+
})
65+
}).then(() => new Promise((resolve) => {
66+
lookup('localhost', (err, address, family) => {
67+
t.equal(err, null, 'no error')
68+
t.equal(address, '127.0.0.1', 'got address')
69+
t.equal(family, 4, 'got family')
70+
t.equal(lookupCalled, 1, 'lookup was still only called once')
71+
resolve()
72+
})
73+
}))
74+
})
75+
76+
t.test('does not cache errors', async (t) => {
77+
let lookupCalled = 0
78+
const fakeLookup = (hostname, options, callback) => {
79+
lookupCalled += 1
80+
if (lookupCalled === 1) {
81+
process.nextTick(callback, new Error('failed'))
82+
return
83+
}
84+
85+
t.match(options, dns.defaultOptions, 'applied default options')
86+
process.nextTick(callback, null, '127.0.0.1', 4)
87+
}
88+
const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
89+
90+
return new Promise((resolve) => {
91+
lookup('localhost', (err, address, family) => {
92+
t.match(err, { message: 'failed' }, 'got the error')
93+
t.equal(lookupCalled, 1, 'lookup was called once')
94+
resolve()
95+
})
96+
}).then(() => new Promise((resolve) => {
97+
lookup('localhost', (err, address, family) => {
98+
t.equal(err, null, 'no error')
99+
t.equal(address, '127.0.0.1', 'got address')
100+
t.equal(family, 4, 'got family')
101+
t.equal(lookupCalled, 2, 'lookup was now called twice')
102+
resolve()
103+
})
104+
})).then(() => new Promise((resolve) => {
105+
lookup('localhost', (err, address, family) => {
106+
t.equal(err, null, 'no error')
107+
t.equal(address, '127.0.0.1', 'got address')
108+
t.equal(family, 4, 'got family')
109+
t.equal(lookupCalled, 2, 'lookup was still only called twice')
110+
resolve()
111+
})
112+
}))
113+
})
114+
115+
t.test('varies when options change', async (t) => {
116+
let lookupCalled = 0
117+
const fakeLookup = (hostname, options, callback) => {
118+
lookupCalled += 1
119+
if (lookupCalled === 1) {
120+
t.match(options, dns.defaultOptions, 'applied default options')
121+
process.nextTick(callback, null, '127.0.0.1', 4)
122+
} else {
123+
t.match(options, { ...dns.defaultOptions, family: 6 }, 'kept family from second lookup')
124+
process.nextTick(callback, null, '::1', 6)
125+
}
126+
}
127+
const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
128+
129+
return new Promise((resolve) => {
130+
lookup('localhost', (err, address, family) => {
131+
t.equal(err, null, 'no error')
132+
t.equal(address, '127.0.0.1', 'got address')
133+
t.equal(family, 4, 'got family')
134+
t.equal(lookupCalled, 1, 'lookup was called once')
135+
resolve()
136+
})
137+
}).then(() => new Promise((resolve) => {
138+
lookup('localhost', { family: 6 }, (err, address, family) => {
139+
t.equal(err, null, 'no error')
140+
t.equal(address, '::1', 'got address')
141+
t.equal(family, 6, 'got family')
142+
t.equal(lookupCalled, 2, 'lookup was called twice')
143+
resolve()
144+
})
145+
}))
146+
})
147+
148+
t.test('lookup can return all results', async (t) => {
149+
let lookupCalled = 0
150+
const fakeLookup = (hostname, options, callback) => {
151+
lookupCalled += 1
152+
t.match(options, { ...dns.defaultOptions, all: true }, 'applied default options')
153+
process.nextTick(callback, null, [{
154+
address: '127.0.0.1', family: 4,
155+
}, {
156+
address: '::1', family: 6,
157+
}])
158+
}
159+
const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
160+
161+
return new Promise((resolve) => {
162+
lookup('localhost', { all: true }, (err, addresses) => {
163+
t.equal(err, null, 'no error')
164+
t.match(addresses, [{
165+
address: '127.0.0.1', family: 4,
166+
}, {
167+
address: '::1', family: 6,
168+
}], 'got all addresses')
169+
t.equal(lookupCalled, 1, 'lookup was called once')
170+
resolve()
171+
})
172+
}).then(() => new Promise((resolve) => {
173+
lookup('localhost', { all: true }, (err, addresses) => {
174+
t.equal(err, null, 'no error')
175+
t.match(addresses, [{
176+
address: '127.0.0.1', family: 4,
177+
}, {
178+
address: '::1', family: 6,
179+
}], 'got all addresses')
180+
t.equal(lookupCalled, 1, 'lookup was called once')
181+
resolve()
182+
})
183+
}))
184+
})
185+
186+
t.test('respects ttl option', async (t) => {
187+
let lookupCalled = 0
188+
const fakeLookup = (hostname, options, callback) => {
189+
lookupCalled += 1
190+
t.match(options, dns.defaultOptions, 'applied default options')
191+
process.nextTick(callback, null, '127.0.0.1', 4)
192+
}
193+
const lookup = dns.getLookup({ ttl: 10, lookup: fakeLookup })
194+
195+
return new Promise((resolve) => {
196+
lookup('localhost', (err, address, family) => {
197+
t.equal(err, null, 'no error')
198+
t.equal(address, '127.0.0.1', 'got address')
199+
t.equal(family, 4, 'got family')
200+
t.equal(lookupCalled, 1, 'lookup was called once')
201+
resolve()
202+
})
203+
}).then(() => new Promise((resolve) => {
204+
lookup('localhost', (err, address, family) => {
205+
t.equal(err, null, 'no error')
206+
t.equal(address, '127.0.0.1', 'got address')
207+
t.equal(family, 4, 'got family')
208+
t.equal(lookupCalled, 1, 'lookup was still only called once')
209+
// delay before the next request to allow the ttl to invalidate
210+
setTimeout(resolve, 15)
211+
})
212+
})).then(() => new Promise((resolve) => {
213+
lookup('localhost', (err, address, family) => {
214+
t.equal(err, null, 'no error')
215+
t.equal(address, '127.0.0.1', 'got address')
216+
t.equal(family, 4, 'got family')
217+
t.equal(lookupCalled, 2, 'lookup was now called twice')
218+
resolve()
219+
})
220+
}))
221+
})

0 commit comments

Comments
 (0)
Please sign in to comment.