Remove the dependency on request

Closes #2792. This reimplements the relevant functionality in the agent-factory.js and http-request.js helper files.

The new HTTP stack implementation fixes several previously-failing XMLHttpRequest tests, where the request module was previously not flexible enough to let us manipulate the headers appropriately.

Note that request remains as an indirect dev dependency of the wd package. This also introduces a dependency on the form-data package which we would like to remove in the future.

lib/jsdom/browser/resources/resource-loader.js

@@ -3,9 +3,10 @@ const fs = require("fs");
 const { fileURLToPath } = require("url");
 const { parseURL } = require("whatwg-url");
 const dataURLFromRecord = require("data-urls").fromURLRecord;
-const request = require("request-promise-native");
-const wrapCookieJarForRequest = require("../../living/helpers/wrap-cookie-jar-for-request");
 const packageVersion = require("../../../../package.json").version;
+const agentFactory = require("../../living/helpers/agent-factory");
+const Request = require("../../living/helpers/http-request");
+
 const IS_BROWSER = Object.prototype.toString.call(process) !== "[object process]";
 
 module.exports = class ResourceLoader {
@@ -67,29 +68,7 @@ module.exports = class ResourceLoader {
     return promise;
   }
 
-  _getRequestOptions({ cookieJar, referrer, accept = "*/*" }) {
-    const requestOptions = {
-      encoding: null,
-      gzip: true,
-      jar: wrapCookieJarForRequest(cookieJar),
-      strictSSL: this._strictSSL,
-      proxy: this._proxy,
-      forever: true,
-      headers: {
-        "User-Agent": this._userAgent,
-        "Accept-Language": "en",
-        "Accept": accept
-      }
-    };
-
-    if (referrer && !IS_BROWSER) {
-      requestOptions.headers.referer = referrer;
-    }
-
-    return requestOptions;
-  }
-
-  fetch(urlString, options = {}) {
+  fetch(urlString, { accept, cookieJar, referrer } = {}) {
     const url = parseURL(urlString);
 
     if (!url) {
@@ -103,8 +82,48 @@ module.exports = class ResourceLoader {
 
       case "http":
       case "https": {
-        const requestOptions = this._getRequestOptions(options);
-        return request(urlString, requestOptions);
+        const agents = agentFactory(this._proxy, this._strictSSL);
+        const headers = {
+          "User-Agent": this._userAgent,
+          "Accept-Language": "en",
+          "Accept-Encoding": "gzip",
+          "Accept": accept || "*/*"
+        };
+        if (referrer && !IS_BROWSER) {
+          headers.Referer = referrer;
+        }
+        const requestClient = new Request(
+          urlString,
+          { followRedirects: true, cookieJar, agents },
+          { headers }
+        );
+        const promise = new Promise((resolve, reject) => {
+          const accumulated = [];
+          requestClient.once("response", res => {
+            promise.response = res;
+            const { statusCode } = res;
+            // TODO This deviates from the spec when it comes to
+            // loading resources such as images
+            if (statusCode < 200 || statusCode > 299) {
+              requestClient.abort();
+              reject(new Error(`Resource was not loaded. Status: ${statusCode}`));
+            }
+          });
+          requestClient.on("data", chunk => {
+            accumulated.push(chunk);
+          });
+          requestClient.on("end", () => resolve(Buffer.concat(accumulated)));
+          requestClient.on("error", reject);
+        });
+        // The method fromURL in lib/api.js crashes without the following four
+        // properties defined on the Promise instance, causing the test suite to halt
+        requestClient.on("end", () => {
+          promise.href = requestClient.currentURL;
+        });
+        promise.abort = requestClient.abort.bind(requestClient);
+        promise.getHeader = name => headers[name] || requestClient.getHeader(name);
+        requestClient.end();
+        return promise;
       }
 
       case "file": {

lib/jsdom/living/helpers/agent-factory.js

@@ -0,0 +1,15 @@
+"use strict";
+const http = require("http");
+const https = require("https");
+const { parse: parseURLToNodeOptions } = require("url");
+const HttpProxyAgent = require("http-proxy-agent");
+const HttpsProxyAgent = require("https-proxy-agent");
+
+module.exports = function agentFactory(proxy, rejectUnauthorized) {
+  const agentOpts = { keepAlive: true, rejectUnauthorized };
+  if (proxy) {
+    const proxyOpts = { ...parseURLToNodeOptions(proxy), ...agentOpts };
+    return { https: new HttpsProxyAgent(proxyOpts), http: new HttpProxyAgent(proxyOpts) };
+  }
+  return { http: new http.Agent(agentOpts), https: new https.Agent(agentOpts) };
+};

lib/jsdom/living/helpers/http-request.js

@@ -0,0 +1,254 @@
+"use strict";
+const http = require("http");
+const https = require("https");
+const { Writable } = require("stream");
+const zlib = require("zlib");
+
+const ver = process.version.replace("v", "").split(".");
+const majorNodeVersion = Number.parseInt(ver[0]);
+
+function abortRequest(clientRequest) {
+  // clientRequest.destroy breaks the test suite for versions 10 and 12,
+  // hence the version check
+  if (majorNodeVersion > 13) {
+    clientRequest.destroy();
+  } else {
+    clientRequest.abort();
+  }
+  clientRequest.removeAllListeners();
+  clientRequest.on("error", () => {});
+}
+
+module.exports = class Request extends Writable {
+  constructor(url, clientOptions, requestOptions) {
+    super();
+    Object.assign(this, clientOptions);
+    this.currentURL = url;
+    this._requestOptions = requestOptions;
+    this.headers = requestOptions.headers;
+    this._ended = false;
+    this._redirectCount = 0;
+    this._requestBodyBuffers = [];
+    this._bufferIndex = 0;
+    this._performRequest();
+  }
+
+  abort() {
+    abortRequest(this._currentRequest);
+    this.emit("abort");
+    this.removeAllListeners();
+  }
+
+  pipeRequest(form) {
+    form.pipe(this._currentRequest);
+  }
+
+  write(data, encoding) {
+    if (data.length > 0) {
+      this._requestBodyBuffers.push({ data, encoding });
+      this._currentRequest.write(data, encoding);
+    }
+  }
+
+  end() {
+    this.emit("request", this._currentRequest);
+    this._ended = true;
+    this._currentRequest.end();
+  }
+
+  setHeader(name, value) {
+    this.headers[name] = value;
+    this._currentRequest.setHeader(name, value);
+  }
+
+  removeHeader(name) {
+    delete this.headers[name];
+    this._currentRequest.removeHeader(name);
+  }
+
+  // Without this method, the test send-redirect-infinite-sync will halt the test suite
+  // TODO: investigate this further and ideally remove
+  toJSON() {
+    const { method, headers } = this._requestOptions;
+    return { uri: new URL(this.currentURL), method, headers };
+  }
+
+  _writeNext(error) {
+    if (this._currentRequest) {
+      if (error) {
+        this.emit("error", error);
+      } else if (this._bufferIndex < this._requestBodyBuffers.length) {
+        const buffer = this._requestBodyBuffers[this._bufferIndex++];
+        if (!this._currentRequest.writableEnded) {
+          this._currentRequest.write(
+            buffer.data,
+            buffer.encoding,
+            this._writeNext.bind(this)
+          );
+        }
+      } else if (this._ended) {
+        this._currentRequest.end();
+      }
+    }
+  }
+
+  _performRequest() {
+    const urlOptions = new URL(this.currentURL);
+    const scheme = urlOptions.protocol;
+    this._requestOptions.agent = this.agents[scheme.substring(0, scheme.length - 1)];
+    const { request } = scheme === "https:" ? https : http;
+    this._currentRequest = request(this.currentURL, this._requestOptions, response => {
+      this._processResponse(response);
+    });
+
+    let cookies;
+    if (this._redirectCount === 0) {
+      this.originalCookieHeader = this.getHeader("Cookie");
+    }
+    if (this.cookieJar) {
+      cookies = this.cookieJar.getCookieStringSync(this.currentURL);
+    }
+    if (cookies && cookies.length) {
+      if (this.originalCookieHeader) {
+        this.setHeader("Cookie", this.originalCookieHeader + "; " + cookies);
+      } else {
+        this.setHeader("Cookie", cookies);
+      }
+    }
+
+    for (const event of ["connect", "error", "socket", "timeout"]) {
+      this._currentRequest.on(event, (...args) => {
+        this.emit(event, ...args);
+      });
+    }
+    if (this._isRedirect) {
+      this._bufferIndex = 0;
+      this._writeNext();
+    }
+  }
+
+  _processResponse(response) {
+    const cookies = response.headers["set-cookie"];
+    if (this.cookieJar && Array.isArray(cookies)) {
+      try {
+        cookies.forEach(cookie => {
+          this.cookieJar.setCookieSync(cookie, this.currentURL, { ignoreError: true });
+        });
+      } catch (e) {
+        this.emit("error", e);
+      }
+    }
+
+    const { statusCode } = response;
+    const { location } = response.headers;
+    // In Node v15, aborting a message with remaining data causes an error to be thrown,
+    // hence the version check
+    const catchResErrors = err => {
+      if (!(majorNodeVersion >= 15 && err.message === "aborted")) {
+        this.emit("error", err);
+      }
+    };
+    response.on("error", catchResErrors);
+    let redirectAddress = null;
+    let resendWithAuth = false;
+    if (typeof location === "string" &&
+      location.length &&
+      this.followRedirects &&
+      statusCode >= 300 &&
+      statusCode < 400) {
+      redirectAddress = location;
+    } else if (statusCode === 401 &&
+      /^Basic /i.test(response.headers["www-authenticate"] || "") &&
+      (this.user && this.user.length)) {
+      this._requestOptions.auth = `${this.user}:${this.pass}`;
+      resendWithAuth = true;
+    }
+    if (redirectAddress || resendWithAuth) {
+      if (++this._redirectCount > 21) {
+        const redirectError = new Error("Maximum number of redirects exceeded");
+        redirectError.code = "ERR_TOO_MANY_REDIRECTS";
+        this.emit("error", redirectError);
+        return;
+      }
+      abortRequest(this._currentRequest);
+      response.destroy();
+      this._isRedirect = true;
+      if (((statusCode === 301 || statusCode === 302) && this._requestOptions.method === "POST") ||
+        (statusCode === 303 && !/^(?:GET|HEAD)$/.test(this._requestOptions.method))) {
+        this._requestOptions.method = "GET";
+        this._requestBodyBuffers = [];
+      }
+      let previousHostName = this._removeMatchingHeaders(/^host$/i);
+      if (!previousHostName) {
+        previousHostName = new URL(this.currentURL).hostname;
+      }
+      const previousURL = this.currentURL;
+      if (!resendWithAuth) {
+        const nextURL = redirectAddress.startsWith("https:") ?
+          new URL(redirectAddress) :
+          new URL(redirectAddress, this.currentURL);
+        if (nextURL.hostname !== previousHostName) {
+          this._removeMatchingHeaders(/^authorization$/i);
+        }
+        this.currentURL = nextURL.toString();
+      }
+      this.headers.Referer = previousURL;
+      this.emit("redirect", response, this.headers, this.currentURL);
+      try {
+        this._performRequest();
+      } catch (cause) {
+        this.emit("error", cause);
+      }
+    } else {
+      let pipeline = response;
+      const acceptEncoding = this.headers["Accept-Encoding"];
+      const requestCompressed = typeof acceptEncoding === "string" &&
+        (acceptEncoding.includes("gzip") || acceptEncoding.includes("deflate"));
+      if (
+        requestCompressed &&
+        this._requestOptions.method !== "HEAD" &&
+        statusCode >= 200 &&
+        statusCode !== 204 &&
+        statusCode !== 304
+      ) {
+        const zlibOptions = {
+          flush: zlib.constants.Z_SYNC_FLUSH,
+          finishFlush: zlib.constants.Z_SYNC_FLUSH
+        };
+        const contentEncoding = (response.headers["content-encoding"] || "identity").trim().toLowerCase();
+        if (contentEncoding === "gzip") {
+          pipeline = zlib.createGunzip(zlibOptions);
+          response.pipe(pipeline);
+        } else if (contentEncoding === "deflate") {
+          pipeline = zlib.createInflate(zlibOptions);
+          response.pipe(pipeline);
+        }
+      }
+      pipeline.removeAllListeners("error");
+      this.emit("response", response, this.currentURL);
+      pipeline.on("data", bytes => this.emit("data", bytes));
+      pipeline.once("end", bytes => this.emit("end", bytes));
+      pipeline.on("error", catchResErrors);
+      pipeline.on("close", () => this.emit("close"));
+      this._requestBodyBuffers = [];
+    }
+  }
+
+  getHeader(key, value) {
+    if (this._currentRequest) {
+      return this._currentRequest.getHeader(key, value);
+    }
+    return null;
+  }
+
+  _removeMatchingHeaders(regex) {
+    let lastValue;
+    for (const header in this.headers) {
+      if (regex.test(header)) {
+        lastValue = this.headers[header];
+        delete this.headers[header];
+      }
+    }
+    return lastValue;
+  }
+};