fix: always send custom User-Agent (#210)

Author: JustinBeckwith

src/index.ts

@@ -59,6 +59,12 @@ interface CrawlOptions {
   rootPath: string;
 }
 
+// Spoof a normal looking User-Agent to keep the servers happy
+export const headers = {
+  'User-Agent':
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
+};
+
 /**
  * Instance class used to perform a crawl job.
  */
@@ -318,10 +324,7 @@ export class LinkChecker extends EventEmitter {
       res = await gaxios.request<string>({
         method: opts.crawl ? 'GET' : 'HEAD',
         url: opts.url.href,
-        headers: {
-          'User-Agent':
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
-        },
+        headers,
         responseType: opts.crawl ? 'text' : 'stream',
         validateStatus: () => true,
         timeout: opts.checkOptions.timeout,
@@ -332,10 +335,7 @@ export class LinkChecker extends EventEmitter {
         res = await gaxios.request<string>({
           method: 'GET',
           url: opts.url.href,
-          headers: {
-            'User-Agent':
-              'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
-          },
+          headers,
           responseType: 'stream',
           validateStatus: () => true,
           timeout: opts.checkOptions.timeout,
@@ -358,6 +358,7 @@ export class LinkChecker extends EventEmitter {
           url: opts.url.href,
           responseType: 'text',
           validateStatus: () => true,
+          headers,
           timeout: opts.checkOptions.timeout,
         });
       }

test/test.ts

@@ -5,7 +5,7 @@ import * as sinon from 'sinon';
 import * as path from 'path';
 import {describe, it, afterEach} from 'mocha';
 
-import {check, LinkState, LinkChecker, CheckOptions} from '../src';
+import {check, LinkState, LinkChecker, CheckOptions, headers} from '../src';
 
 nock.disableNetConnect();
 nock.enableNetConnect('localhost');
@@ -416,4 +416,24 @@ describe('linkinator', () => {
       /returned 0 results/
     );
   });
+
+  it('should always send a human looking User-Agent', async () => {
+    const scopes = [
+      nock('http://fake.local')
+        .get('/', undefined, {reqheaders: headers})
+        .replyWithFile(200, 'test/fixtures/local/index.html', {
+          'Content-Type': 'text/html; charset=UTF-8',
+        }),
+      nock('http://fake.local')
+        .get('/page2.html', undefined, {reqheaders: headers})
+        .replyWithFile(200, 'test/fixtures/local/page2.html', {
+          'Content-Type': 'text/html; charset=UTF-8',
+        }),
+    ];
+    const results = await check({
+      path: 'http://fake.local',
+    });
+    assert.ok(results.passed);
+    scopes.forEach(x => x.done());
+  });
 });