VM: Add comparison testing between branchs for State Test Runner (#1634)

* Add diff tester and performance

* update script

* Simplify script

* Readme updates

* move start

* Update stashing logic in script

packages/vm/DEVELOPER.md

@@ -144,6 +144,27 @@ python utils/diffTestOutput.py output-wip-byzantium.txt output-master.txt
 
 An extremely rich and powerful toolbox is the [evmlab](https://github.com/holiman/evmlab) from `holiman`, both for debugging and creating new test cases or example data.
 
+## Git Branch Performance Testing
+
+The [`diffTester`](./scripts/diffTester.sh) script can be used to do simple comparative performance testing of changes made targeting the VM.  This script allows you to run a single State test a specified number of times on two different branches and reports the average time of the test run for each branch.  While not statistically rigorous, it gives you a quick sense of how a specific change (or set of changes) may impact VM performance on a given area that is covered by one specific test.  Run this script from `[monorepo-root]/packages/vm` as below:
+```sh
+./scripts/diffTester.sh -b git-branch-you-want-to-test -t "path/to/my/favorite/state/test.json" -r [the number of times to run the test]
+```
+
+and it will produce output like for the `git-branch-you-want-to-test` and then whatever git branch you are currently on:
+```sh
+TAP version 13
+# GeneralStateTests
+# file: path/to/my/favorite/state/test.json test: test
+ok 1 [ 3.785 secs ] the state roots should match (successful tx run)
+ok 2 [ 1.228 secs ] the state roots should match (successful tx run)
+ok 3 [ 1.212 secs ] the state roots should match (successful tx run)
+ok 4 [ 1.306 secs ] the state roots should match (successful tx run)
+ok 5 [ 1.472 secs ] the state roots should match (successful tx run)
+# Average test run: 1.801 s
+```
+
+Note: this script runs by actually checking out the targeted branch, running the test, and then switching back to your current branch, running the test again, and then restoring any changes you had in the current branch.  For best results, you shuld run this test while you currently have `master` checked out.
 ## Profiling
 
 [Clinic](https://github.com/nearform/node-clinic) allows profiling the VM in the node environment. It supports various profiling methods, among them is [flame](https://github.com/nearform/node-clinic-flame) which can be used for generating flamegraphs to highlight bottlenecks and hot paths. As an example, to generate a flamegraph for the VM blockchain tests, you can run:

packages/vm/scripts/diffTester.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+usage() {
+    echo 'Usage: ./diffTester.sh [-b branch] [-t "path/to/my/test.json"] [-r 5] [-f London]'
+}
+
+FORK="London"
+STASH="f"
+while getopts "b:t:r:f::" c
+do
+    case $c in
+        b) BRANCH=$OPTARG ;;
+        t) TEST=$OPTARG ;;
+        r) REPS=$OPTARG ;;
+        f) FORK=$OPTARG ;;
+        h|?) usage ;;
+    esac
+done
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+if ! git status | grep -q "nothing to commit"
+then    
+    git stash
+    STASH="t"
+fi
+
+git checkout $BRANCH
+
+npm run tester -- --state --fork=$FORK --customStateTest=$TEST --reps=$REPS
+
+git checkout $CURRENT_BRANCH
+
+npm run tester -- --state --fork=$FORK --customStateTest=$TEST --reps=$REPS
+
+echo "stash" $STASH
+if [ "$STASH" = "t" ]
+then
+    git stash pop
+fi

packages/vm/tests/tester/index.ts

@@ -39,6 +39,7 @@ import blockchainTestsRunner from './runners/BlockchainTestsRunner'
  * --debug: enable BlockchainTests debugger (compares post state against the expected post state)
  * --expected-test-amount: (optional) if present, check after tests are ran if at least this amount of tests have passed (inclusive)
  * --verify-test-amount-alltests: if this is passed, get the expected amount from tests and verify afterwards if this is the count of tests (expects tests are ran with default settings)
+ * --reps: if passed, each test case will be run the number of times indicated
  */
 
 const argv = minimist(process.argv.slice(2))
@@ -90,6 +91,7 @@ async function runTests() {
   runnerArgs.gasLimit = argv.gas // GeneralStateTests
   runnerArgs.value = argv.value // GeneralStateTests
   runnerArgs.debug = argv.debug // BlockchainTests
+  runnerArgs.reps = argv.reps // test repetitions
 
   let expectedTests: number | undefined
   if (argv['verify-test-amount-alltests']) {

packages/vm/tests/tester/runners/GeneralStateTestsRunner.ts

@@ -60,14 +60,13 @@ function parseTestCases(
 }
 
 async function runTestCase(options: any, testData: any, t: tape.Test) {
-  const begin = Date.now()
   let VM
   if (options.dist) {
     VM = require('../../../dist').default
   } else {
     VM = require('../../../src').default
   }
-
+  const begin = Date.now()
   const state = new Trie()
   const hardfork = options.forkConfigVM
 
@@ -136,6 +135,7 @@ async function runTestCase(options: any, testData: any, t: tape.Test) {
   const timeSpent = `${(end - begin) / 1000} secs`
 
   t.ok(stateRootsAreEqual, `[ ${timeSpent} ] the state roots should match (${execInfo})`)
+  return parseFloat(timeSpent)
 }
 
 export default async function runStateTest(options: any, testData: any, t: tape.Test) {
@@ -152,7 +152,15 @@ export default async function runStateTest(options: any, testData: any, t: tape.
       return
     }
     for (const testCase of testCases) {
-      await runTestCase(options, testCase, t)
+      if (options.reps) {
+        let totalTimeSpent = 0
+        for (let x = 0; x < options.reps; x++) {
+          totalTimeSpent += await runTestCase(options, testCase, t)
+        }
+        t.comment(`Average test run: ${(totalTimeSpent / options.reps).toLocaleString()} s`)
+      } else {
+        await runTestCase(options, testCase, t)
+      }
     }
   } catch (e: any) {
     console.log(e)