Allow custom upstreams to be specified (#568)

This adds back Miniflare 2's `upstream` option, which allows you to
customise the origin of incoming requests to the Worker. This is
useful when implementing proxy servers with Workers. Wrangler exposes
this option as the `--local-upstream` flag, so we'd like to support
this in Miniflare 3 to avoid regressing when we graduate.

Closes cloudflare/workers-sdk#2821
Closes DEVX-628

Author: mrbbot

packages/tre/src/plugins/core/index.ts

@@ -71,6 +71,7 @@ export const CoreSharedOptionsSchema = z.object({
   log: z.instanceof(Log).optional(),
   timers: z.custom<Timers>().optional(),
 
+  upstream: z.string().optional(),
   // TODO: add back validation of cf object
   cf: z.union([z.boolean(), z.string(), z.record(z.any())]).optional(),
 
@@ -345,6 +346,12 @@ export function getGlobalServices({
       service: { name: getUserServiceName(name) },
     })),
   ];
+  if (sharedOptions.upstream !== undefined) {
+    serviceEntryBindings.push({
+      name: CoreBindings.TEXT_UPSTREAM_URL,
+      text: sharedOptions.upstream,
+    });
+  }
   if (sharedOptions.liveReload) {
     const liveReloadScript = LIVE_RELOAD_SCRIPT_TEMPLATE(loopbackPort);
     serviceEntryBindings.push({
@@ -371,9 +378,7 @@ export function getGlobalServices({
     {
       name: "internet",
       network: {
-        // Can't use `["public", "private"]` here because of
-        // https://github.com/cloudflare/workerd/issues/62
-        allow: ["0.0.0.0/0"],
+        allow: ["public", "private"],
         deny: [],
         tlsOptions: { trustBrowserCas: true },
       },

packages/tre/src/workers/core/constants.ts

@@ -11,6 +11,7 @@ export const CoreBindings = {
   SERVICE_USER_ROUTE_PREFIX: "MINIFLARE_USER_ROUTE_",
   SERVICE_USER_FALLBACK: "MINIFLARE_USER_FALLBACK",
   TEXT_CUSTOM_SERVICE: "MINIFLARE_CUSTOM_SERVICE",
+  TEXT_UPSTREAM_URL: "MINIFLARE_UPSTREAM_URL",
   JSON_CF_BLOB: "CF_BLOB",
   JSON_ROUTES: "MINIFLARE_ROUTES",
   JSON_LOG_LEVEL: "MINIFLARE_LOG_LEVEL",

packages/tre/src/workers/core/entry.worker.ts

@@ -11,6 +11,7 @@ type Env = {
   [CoreBindings.SERVICE_LOOPBACK]: Fetcher;
   [CoreBindings.SERVICE_USER_FALLBACK]: Fetcher;
   [CoreBindings.TEXT_CUSTOM_SERVICE]: string;
+  [CoreBindings.TEXT_UPSTREAM_URL]?: string;
   [CoreBindings.JSON_CF_BLOB]: IncomingRequestCfProperties;
   [CoreBindings.JSON_ROUTES]: WorkerRoute[];
   [CoreBindings.JSON_LOG_LEVEL]: LogLevel;
@@ -26,7 +27,17 @@ function getUserRequest(
   env: Env
 ) {
   const originalUrl = request.headers.get(CoreHeaders.ORIGINAL_URL);
-  request = new Request(originalUrl ?? request.url, {
+  const upstreamUrl = env[CoreBindings.TEXT_UPSTREAM_URL];
+  let url = new URL(originalUrl ?? request.url);
+  if (upstreamUrl !== undefined) {
+    // If a custom `upstream` was specified, make sure the URL starts with it
+    let path = url.pathname + url.search;
+    // Remove leading slash, so we resolve relative to `upstream`'s path
+    if (path.startsWith("/")) path = path.substring(1);
+    url = new URL(path, upstreamUrl);
+  }
+
+  request = new Request(url, {
     method: request.method,
     headers: request.headers,
     cf: request.cf ?? env[CoreBindings.JSON_CF_BLOB],

packages/tre/test/index.spec.ts

@@ -15,6 +15,7 @@ import {
   MessageEvent as StandardMessageEvent,
   WebSocketServer,
 } from "ws";
+import { useServer } from "./test-shared";
 
 test("Miniflare: validates options", async (t) => {
   // Check empty workers array rejected
@@ -231,3 +232,21 @@ test("Miniflare: custom service binding to another Miniflare instance", async (t
     body: null,
   });
 });
+
+test("Miniflare: uses custom upstream as origin", async (t) => {
+  const upstream = await useServer(t, (req, res) => {
+    res.end(`upstream: ${new URL(req.url ?? "", "http://upstream")}`);
+  });
+  const mf = new Miniflare({
+    upstream: new URL("/extra/", upstream.http.toString()).toString(),
+    modules: true,
+    script: `export default {
+      fetch(request) {
+        return fetch(request);
+      }
+    }`,
+  });
+  // Check rewrites protocol, hostname, and port, but keeps pathname and query
+  const res = await mf.dispatchFetch("https://random:0/path?a=1");
+  t.is(await res.text(), "upstream: http://upstream/extra/path?a=1");
+});