feat: async peerstore backed by datastores (#1058)

We have a peerstore that keeps all data for all observed peers in memory with no eviction.

This is fine when you don't discover many peers but when using the DHT you encounter a significant number of peers so our peer storage grows and grows over time.

We have a persistent peer store, but it just periodically writes peers into the datastore to be read at startup, still keeping them in memory.

It also means a restart doesn't give you any temporary reprieve from the memory leak as the previously observed peer data is read into memory at startup.

This change refactors the peerstore to use a datastore by default, reading and writing peer info as it arrives.  It can be configured with a MemoryDatastore if desired.

It was necessary to change the peerstore and *book interfaces to be asynchronous since the datastore api is asynchronous.

BREAKING CHANGE: `libp2p.handle`, `libp2p.registrar.register` and the peerstore methods have become async

Author: achingbrain

.github/workflows/examples.yml

@@ -12,29 +12,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
         node: [16]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
           node-version: 16
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-           npm install
-           npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
 
   check:
     needs: build
@@ -43,23 +28,8 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-node@v2
       with:
-        node-version: 16
-    - uses: actions/cache@v2
-      id: cache
-      env:
-        CACHE_NAME: cache-node-modules
-      with:
-        path: |
-          ~/.cache
-          ~/.npm
-          ./node_modules
-          ./dist
-        key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-    - name: Install Dependencies
-      if: steps.cache.outputs.cache-hit != 'true'
-      run: |
-        npm install
-        npm run build
+        node-version: lts/*
+    - uses: ipfs/aegir/actions/cache-node-modules@master
     - run: npx aegir lint
     - run: npx aegir dep-check
     - uses: ipfs/aegir/actions/bundle-size@master
@@ -80,23 +50,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node }}
-      - uses: actions/cache@v2
-        id: cache
-        if: matrix.os != 'windows-latest'
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:node -- --cov --bail
       - uses: codecov/codecov-action@v1
   test-chrome:
@@ -107,22 +61,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:browser -- -t browser -t webworker --bail
   test-firefox:
     needs: check
@@ -132,22 +71,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:browser -- -t browser -t webworker --bail -- --browser firefox
   test-ts:
     needs: check
@@ -157,22 +81,7 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:ts
   test-interop:
     needs: check
@@ -182,20 +91,5 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: lts/*
-      - uses: actions/cache@v2
-        id: cache
-        env:
-          CACHE_NAME: cache-node-modules
-        with:
-          path: |
-            ~/.cache
-            ~/.npm
-            ./node_modules
-            ./dist
-          key: ${{ runner.os }}-build-${{ env.CACHE_NAME }}-${{ github.event.pull_request.head.sha }}
-      - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          npm install
-          npm run build
+      - uses: ipfs/aegir/actions/cache-node-modules@master
       - run: npm run test:interop -- --bail -- --exit