Remove legacy storage system

Now that all gateways have been migrated to the new storage system,
we aren't using the legacy one. Also removes remote storage, which
will need rethinking with the new storage system.

Author: mrbbot

packages/tre/README.md

@@ -34,9 +34,6 @@ await mf.dispose();
 
 ## API
 
-> :warning: Features marked **(Experimental)** may change at any point and are
-> not subject to semantic versioning guarantees.
-
 ### `type Awaitable<T>`
 
 `T | Promise<T>`
@@ -121,14 +118,6 @@ Represents where data should be persisted, if anywhere.
   - If the protocol is `file:`, data will be stored on the file-system, in the
     specified directory (e.g. `file:///path/to/directory`). If the `unsanitise`
     search parameter is `true`, path sanitisation will be disabled.
-  - If the protocol is `sqlite:`, data will be stored in an SQLite database, at
-    the specified path (e.g. `sqlite:///path/to/db.sqlite`).
-  - **(Experimental)** If the protocol is `remote:`, data will be read/written
-    from/to real data stores on the Cloudflare network. By default, this will
-    cache data in-memory, but the `cache` search parameter can be set to a
-    URL-encoded persistence string to customise this. Note, this feature is only
-    supported for KV namespaces at the moment, and requires the
-    `cloudflareFetch` option to be set.
 - Otherwise, if this is just a regular `string`, data will be stored on the
   file-system, using the value as the directory path.
 
@@ -433,14 +422,6 @@ Options shared between all Workers/"nanoservices".
   automatically reloads the page in-browser whenever the Miniflare instance's
   options are updated.
 
-- **(Experimental)**
-  `cloudflareFetch?: (resource: string, searchParams?: URLSearchParams, init?: RequestInit) => Awaitable<Response>`
-
-  Authenticated `fetch` used by `remote:` storage to communicate with the
-  Cloudflare API. `https://api.cloudflare.com/client/v4/accounts/<account_id>/`
-  should be prepended to `resource` to form the request URL. Appropriate
-  authorization headers should also be added.
-
 #### Cache, Durable Objects, KV, R2 and D1
 
 - `cachePersist?: Persistence`

packages/tre/src/index.ts

@@ -418,10 +418,8 @@ export class Miniflare {
           this.#log,
           this.#timers,
           this.dispatchFetch,
-          this.#sharedOpts.core.cloudflareFetch,
           key,
-          plugin.gateway,
-          plugin.remoteStorage
+          plugin.gateway
         );
         const router = new plugin.router(this.#log, gatewayFactory);
         // @ts-expect-error this.#gatewayFactories[key] could be any plugin's
@@ -889,7 +887,7 @@ export class Miniflare {
   ): NewStorage {
     const factory = this.#gatewayFactories[plugin];
     assert(factory !== undefined);
-    return factory.getStorage(namespace, persist).getNewStorage();
+    return factory.getStorage(namespace, persist);
   }
 
   async dispose(): Promise<void> {
@@ -914,6 +912,5 @@ export * from "./http";
 export * from "./plugins";
 export * from "./runtime";
 export * from "./shared";
-export * from "./storage";
 export * from "./storage2";
 export * from "./workers";

packages/tre/src/plugins/cache/gateway.ts

@@ -5,11 +5,11 @@ import { ReadableStream, TransformStream } from "stream/web";
 import CachePolicy from "http-cache-semantics";
 import { Headers, HeadersInit, Request, Response, fetch } from "../../http";
 import { DeferredPromise, Log, Timers } from "../../shared";
-import { Storage } from "../../storage";
 import {
   InclusiveRange,
   KeyValueStorage,
   MultipartReadableStream,
+  NewStorage,
 } from "../../storage2";
 import { isSitesRequest } from "../kv";
 import { _parseRanges } from "../shared";
@@ -232,10 +232,9 @@ export class CacheGateway {
 
   constructor(
     private readonly log: Log,
-    legacyStorage: Storage,
+    storage: NewStorage,
     private readonly timers: Timers
   ) {
-    const storage = legacyStorage.getNewStorage();
     this.storage = new KeyValueStorage(storage, timers);
   }
 

packages/tre/src/plugins/core/index.ts

@@ -23,7 +23,6 @@ import { CoreBindings, CoreHeaders } from "../../workers";
 import { getCacheServiceName } from "../cache";
 import { DURABLE_OBJECTS_STORAGE_SERVICE_NAME } from "../do";
 import {
-  CloudflareFetchSchema,
   HEADER_CF_BLOB,
   Plugin,
   SERVICE_LOOPBACK,
@@ -71,7 +70,6 @@ export const CoreSharedOptionsSchema = z.object({
 
   log: z.instanceof(Log).optional(),
   timers: z.custom<Timers>().optional(),
-  cloudflareFetch: CloudflareFetchSchema.optional(),
 
   // TODO: add back validation of cf object
   cf: z.union([z.boolean(), z.string(), z.record(z.any())]).optional(),

packages/tre/src/plugins/d1/gateway.ts

@@ -7,7 +7,7 @@ import type { Database as DatabaseType } from "better-sqlite3";
 import { z } from "zod";
 import { Response } from "../../http";
 import { HttpError, Log } from "../../shared";
-import { Storage } from "../../storage";
+import { NewStorage } from "../../storage2";
 import splitSqlQuery from "./splitter";
 
 export const D1ValueSchema = z.union([
@@ -153,8 +153,7 @@ interface ChangesLastRowResult {
 export class D1Gateway {
   private readonly db: DatabaseType;
 
-  constructor(private readonly log: Log, legacyStorage: Storage) {
-    const storage = legacyStorage.getNewStorage();
+  constructor(private readonly log: Log, storage: NewStorage) {
     this.db = storage.db;
   }
 

packages/tre/src/plugins/kv/gateway.ts

@@ -7,8 +7,7 @@ import {
   millisToSeconds,
   secondsToMillis,
 } from "../../shared";
-import { Storage } from "../../storage";
-import { KeyValueStorage } from "../../storage2";
+import { KeyValueStorage, NewStorage } from "../../storage2";
 import {
   MAX_KEY_SIZE,
   MAX_LIST_KEYS,
@@ -158,10 +157,9 @@ export class KVGateway {
 
   constructor(
     private readonly log: Log,
-    legacyStorage: Storage,
+    storage: NewStorage,
     private readonly timers: Timers
   ) {
-    const storage = legacyStorage.getNewStorage();
     this.storage = new KeyValueStorage(storage, timers);
   }
 

packages/tre/src/plugins/kv/index.ts

@@ -8,7 +8,6 @@ import {
 } from "../shared";
 import { KV_PLUGIN_NAME } from "./constants";
 import { KVGateway } from "./gateway";
-import { KVRemoteStorage } from "./remote";
 import { KVRouter } from "./router";
 import { SitesOptions, getSitesBindings, getSitesService } from "./sites";
 
@@ -39,7 +38,6 @@ export const KV_PLUGIN: Plugin<
 > = {
   gateway: KVGateway,
   router: KVRouter,
-  remoteStorage: KVRemoteStorage,
   options: KVOptionsSchema,
   sharedOptions: KVSharedOptionsSchema,
   async getBindings(options) {

packages/tre/src/plugins/kv/remote.ts

@@ -7,7 +7,7 @@ import { stringify } from "devalue";
 import { Colorize, bold, green, grey, red, reset, yellow } from "kleur/colors";
 import { z } from "zod";
 import { Log, Timers } from "../../shared";
-import { Storage } from "../../storage";
+import { NewStorage } from "../../storage2";
 import { CoreHeaders, structuredSerializableReducers } from "../../workers";
 import { DispatchFetch, QueueConsumer } from "../shared";
 
@@ -99,7 +99,7 @@ export class QueuesGateway {
 
   constructor(
     private readonly log: Log,
-    _storage: Storage,
+    _storage: NewStorage,
     private readonly timers: Timers,
     private readonly queueName: string,
     private readonly dispatchFetch: DispatchFetch

packages/tre/src/plugins/queues/gateway.ts

@@ -11,7 +11,6 @@ import {
   maybeApply,
   prefixError,
 } from "../../shared";
-import { Storage } from "../../storage";
 import {
   BlobId,
   InclusiveRange,
@@ -569,10 +568,10 @@ export class R2Gateway {
 
   constructor(
     private readonly log: Log,
-    legacyStorage: Storage,
+    storage: NewStorage,
     private readonly timers: Timers
   ) {
-    this.#storage = legacyStorage.getNewStorage();
+    this.#storage = storage;
     this.#storage.db.pragma("case_sensitive_like = TRUE");
     this.#storage.db.exec(SQL_SCHEMA);
     this.#stmts = sqlStmts(this.#storage.db);

packages/tre/src/plugins/r2/gateway.ts

@@ -11,12 +11,10 @@ import {
   sanitisePath,
 } from "../../shared";
 import {
-  FileStorage,
-  MemoryStorage,
-  RemoteStorage,
-  SqliteStorage,
-  Storage,
-} from "../../storage";
+  NewStorage,
+  createFileStorage,
+  createMemoryStorage,
+} from "../../storage2";
 
 // TODO: explain why persist passed as header, want options set to be atomic,
 //  if set gateway before script update, may be using new persist before new script
@@ -42,21 +40,13 @@ export type CloudflareFetch = z.infer<typeof CloudflareFetchSchema>;
 export interface GatewayConstructor<Gateway> {
   new (
     log: Log,
-    storage: Storage,
+    storage: NewStorage,
     timers: Timers,
     namespace: string,
     dispatchFetch: DispatchFetch
   ): Gateway;
 }
 
-export interface RemoteStorageConstructor {
-  new (
-    cache: Storage,
-    cloudflareFetch: CloudflareFetch,
-    namespace: string
-  ): RemoteStorage;
-}
-
 export const DEFAULT_PERSIST_ROOT = ".mf";
 
 export const PARAM_FILE_UNSANITISE = "unsanitise";
@@ -69,30 +59,25 @@ export function maybeParseURL(url: Persistence): URL | undefined {
 }
 
 export class GatewayFactory<Gateway> {
-  readonly #memoryStorages = new Map<string, MemoryStorage>();
+  readonly #memoryStorages = new Map<string, NewStorage>();
   readonly #gateways = new Map<string, [Persistence, Gateway]>();
 
   constructor(
     private readonly log: Log,
     private readonly timers: Timers,
     private readonly dispatchFetch: DispatchFetch,
-    private readonly cloudflareFetch: CloudflareFetch | undefined,
     private readonly pluginName: string,
-    private readonly gatewayClass: GatewayConstructor<Gateway>,
-    private readonly remoteStorageClass?: RemoteStorageConstructor
+    private readonly gatewayClass: GatewayConstructor<Gateway>
   ) {}
 
   #getMemoryStorage(namespace: string) {
     let storage = this.#memoryStorages.get(namespace);
     if (storage !== undefined) return storage;
-    this.#memoryStorages.set(
-      namespace,
-      (storage = new MemoryStorage(undefined, this.timers.now))
-    );
+    this.#memoryStorages.set(namespace, (storage = createMemoryStorage()));
     return storage;
   }
 
-  getStorage(namespace: string, persist: Persistence): Storage {
+  getStorage(namespace: string, persist: Persistence): NewStorage {
     // If persistence is disabled, use memory storage
     if (persist === undefined || persist === false) {
       return this.#getMemoryStorage(namespace);
@@ -109,47 +94,20 @@ export class GatewayFactory<Gateway> {
       }
       if (url.protocol === "file:") {
         const root = path.join(fileURLToPath(url), sanitisedNamespace);
-        const unsanitise =
-          url.searchParams.get(PARAM_FILE_UNSANITISE) === "true";
-        return new FileStorage(root, !unsanitise, this.timers.now);
-      } else if (url.protocol === "sqlite:") {
-        return new SqliteStorage(
-          url.pathname,
-          sanitisedNamespace,
-          this.timers.now
-        );
-      }
-      // TODO: support Redis storage?
-      if (url.protocol === "remote:") {
-        const { cloudflareFetch, remoteStorageClass } = this;
-        if (cloudflareFetch === undefined) {
-          throw new MiniflareCoreError(
-            "ERR_PERSIST_REMOTE_UNAUTHENTICATED",
-            "Authenticated Cloudflare API `cloudflareFetch` option not provided but required for remote storage"
-          );
-        }
-        if (remoteStorageClass === undefined) {
-          throw new MiniflareCoreError(
-            "ERR_PERSIST_REMOTE_UNSUPPORTED",
-            `The "${this.pluginName}" plugin does not support remote storage`
-          );
-        }
-        const cachePersist = url.searchParams.get("cache") ?? undefined;
-        const cache = this.getStorage(namespace, cachePersist);
-        return new remoteStorageClass(cache, cloudflareFetch, namespace);
+        return createFileStorage(root);
       }
       throw new MiniflareCoreError(
         "ERR_PERSIST_UNSUPPORTED",
         `Unsupported "${url.protocol}" persistence protocol for storage: ${url.href}`
       );
     }
 
-    // Otherwise, fallback to sanitised file storage
+    // Otherwise, fallback to file storage
     const root =
       persist === true
         ? path.join(DEFAULT_PERSIST_ROOT, this.pluginName, sanitisedNamespace)
         : path.join(persist, sanitisedNamespace);
-    return new FileStorage(root, undefined, this.timers.now);
+    return createFileStorage(root);
   }
 
   get(namespace: string, persist: Persistence): Gateway {

packages/tre/src/plugins/shared/gateway.ts

@@ -1,7 +1,7 @@
 import { z } from "zod";
 import { Service, Worker_Binding, Worker_Module } from "../../runtime";
 import { Awaitable, Log, OptionalZodTypeOf } from "../../shared";
-import { GatewayConstructor, RemoteStorageConstructor } from "./gateway";
+import { GatewayConstructor } from "./gateway";
 import { RouterConstructor } from "./router";
 
 // Maps **service** names to the Durable Object class names exported by them
@@ -70,11 +70,10 @@ export type Plugin<
     ? { sharedOptions?: undefined }
     : { sharedOptions: SharedOptions }) &
   (Gateway extends undefined
-    ? { gateway?: undefined; router?: undefined; remoteStorage?: undefined }
+    ? { gateway?: undefined; router?: undefined }
     : {
         gateway: GatewayConstructor<Gateway>;
         router: RouterConstructor<Gateway>;
-        remoteStorage?: RemoteStorageConstructor;
       });
 
 export function namespaceEntries(

packages/tre/src/plugins/shared/index.ts

@@ -3,10 +3,10 @@ import crypto from "crypto";
 import path from "path";
 import { text } from "stream/consumers";
 import {
-  FileStorage,
   HeadersInit,
   KeyValueStorage,
   LogLevel,
+  createFileStorage,
 } from "@miniflare/tre";
 import { miniflareTest, useTmp } from "../../test-shared";
 
@@ -406,15 +406,8 @@ test.serial("operations log warning on workers.dev subdomain", async (t) => {
 test.serial("operations persist cached data", async (t) => {
   // Create new temporary file-system persistence directory
   const tmp = await useTmp(t);
-  const clock = () => t.context.timers.timestamp;
-  // TODO(soon): clean up this mess once we've migrated all gateways
-  const legacyStorage = new FileStorage(
-    path.join(tmp, "default"),
-    undefined,
-    clock
-  );
-  const newStorage = legacyStorage.getNewStorage();
-  const kvStorage = new KeyValueStorage(newStorage, t.context.timers);
+  const storage = createFileStorage(path.join(tmp, "default"));
+  const kvStorage = new KeyValueStorage(storage, t.context.timers);
 
   // Set option, then reset after test
   await t.context.setOptions({ cachePersist: tmp });

packages/tre/src/storage/file.ts

@@ -6,7 +6,7 @@ import type {
   D1PreparedStatement,
   D1Result,
 } from "@cloudflare/workers-types/experimental";
-import { FileStorage, Miniflare, MiniflareOptions } from "@miniflare/tre";
+import { Miniflare, MiniflareOptions, createFileStorage } from "@miniflare/tre";
 import Database from "better-sqlite3";
 import {
   MiniflareTestContext,
@@ -468,10 +468,8 @@ test.serial("operations persist D1 data", async (t) => {
 
   // Create new temporary file-system persistence directory
   const tmp = await useTmp(t);
-  // TODO(soon): clean up this mess once we've migrated all gateways
-  const legacyStorage = new FileStorage(path.join(tmp, "db"));
-  const newStorage = legacyStorage.getNewStorage();
-  const sqliteDb = newStorage.db;
+  const storage = createFileStorage(path.join(tmp, "db"));
+  const sqliteDb = storage.db;
 
   // Set option, then reset after test
   await t.context.setOptions({ ...opts, d1Persist: tmp });

packages/tre/src/storage/index.ts

@@ -7,17 +7,19 @@ import {
   KVGatewayListOptions,
   KVGatewayListResult,
   KeyValueStorage,
-  MemoryStorage,
   NoOpLog,
+  createMemoryStorage,
 } from "@miniflare/tre";
 import anyTest, { Macro, TestFn, ThrowsExpectation } from "ava";
-import {
-  TIME_EXPIRED,
-  TIME_EXPIRING,
-  TIME_NOW,
-  createJunkStream,
-  testTimers,
-} from "../../test-shared";
+import { TestTimers, createJunkStream } from "../../test-shared";
+
+// Stored expiration value to signal an expired key.
+export const TIME_EXPIRED = 500;
+// Time in seconds the testClock always returns:
+// TIME_EXPIRED < TIME_NOW < TIME_EXPIRING
+export const TIME_NOW = 1000;
+// Stored expiration value to signal a key that will expire in the future.
+export const TIME_EXPIRING = 1500;
 
 interface Context {
   storage: KeyValueStorage;
@@ -27,11 +29,10 @@ interface Context {
 const test = anyTest as TestFn<Context>;
 
 test.beforeEach((t) => {
-  // TODO(soon): clean up this mess once we've migrated all gateways
-  const legacyStorage = new MemoryStorage(undefined, testTimers.now);
-  const newStorage = legacyStorage.getNewStorage();
-  const gateway = new KVGateway(new NoOpLog(), legacyStorage, testTimers);
-  const kvStorage = new KeyValueStorage(newStorage, testTimers);
+  const storage = createMemoryStorage();
+  const timers = new TestTimers();
+  const gateway = new KVGateway(new NoOpLog(), storage, timers);
+  const kvStorage = new KeyValueStorage(storage, timers);
   t.context = { storage: kvStorage, gateway };
 });
 

packages/tre/src/storage/local.ts

@@ -28,7 +28,6 @@ import type {
 } from "@cloudflare/workers-types/experimental";
 import {
   File,
-  FileStorage,
   FormData,
   Headers,
   Miniflare,
@@ -39,6 +38,7 @@ import {
   R2Gateway,
   Response,
   TypedDatabase,
+  createFileStorage,
   viewToArray,
   viewToBuffer,
 } from "@miniflare/tre";
@@ -1346,16 +1346,15 @@ test.serial("operations persist stored data", async (t) => {
 
   // Create new temporary file-system persistence directory
   const tmp = await useTmp(t);
-  const legacyStorage = new FileStorage(path.join(tmp, "bucket"));
-  const newStorage = legacyStorage.getNewStorage();
+  const storage = createFileStorage(path.join(tmp, "bucket"));
 
   // Set option, then reset after test
   await t.context.setOptions({ ...opts, r2Persist: tmp });
   t.teardown(() => t.context.setOptions(opts));
 
   // Check put respects persist
   await r2.put("key", "value");
-  const stmtListByNs = newStorage.db.prepare<{ ns: string }, { key: string }>(
+  const stmtListByNs = storage.db.prepare<{ ns: string }, { key: string }>(
     "SELECT key FROM _mf_objects WHERE key LIKE :ns || '%'"
   );
   let stored = stmtListByNs.all({ ns });

packages/tre/src/storage/memory.ts

@@ -25,7 +25,6 @@ class TestRouter extends Router<TestGateway> {
         log,
         defaultTimers,
         () => assert.fail("dispatchFetch not implemented"),
-        undefined,
         "test",
         TestGateway
       )

packages/tre/src/storage/sqlite.ts

@@ -3,7 +3,7 @@ import fs from "fs/promises";
 import path from "path";
 import { ReadableStream } from "stream/web";
 import { TextDecoder, TextEncoder } from "util";
-import { Timers, sanitisePath, unwrapBYOBRequest } from "@miniflare/tre";
+import { sanitisePath, unwrapBYOBRequest } from "@miniflare/tre";
 import { ExecutionContext } from "ava";
 
 const encoder = new TextEncoder();
@@ -16,25 +16,6 @@ export function utf8Decode(encoded?: Uint8Array): string {
   return decoder.decode(encoded);
 }
 
-// Stored expiration value to signal an expired key. Storages using actual
-// time should interpret this as the current time.
-export const TIME_EXPIRED = 500;
-// Time in seconds the testClock always returns:
-// TIME_EXPIRED < TIME_NOW < TIME_EXPIRING
-export const TIME_NOW = 750;
-// Stored expiration value to signal a key that will expire in the future.
-// Storages using actual time should interpret this as the current time + 1hr.
-// Tests will check the expiry is within 120s of this.
-export const TIME_EXPIRING = 1000;
-
-// TODO(soon): remove once we remove the old storage system
-export const testTimers: Timers = {
-  now: () => TIME_NOW * 1000,
-  setTimeout,
-  clearTimeout,
-  queueMicrotask,
-};
-
 const tmpRoot = path.resolve(".tmp");
 export async function useTmp(t: ExecutionContext): Promise<string> {
   const filePath = path.join(