chore: initial codebase refactor (#3517)

.eslintignore

@@ -0,0 +1,2 @@
+dist
+__fixtures__

.github/workflows/ci.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - main
       - next
-      - "test/*"
   pull_request:
     types: [assigned, opened, synchronize, reopened, labeled]
 
@@ -17,6 +16,7 @@ env:
   NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
   NX_CLOUD_DISTRIBUTED_EXECUTION: true
   NX_BRANCH: ${{ github.event.number || github.ref_name }}
+  NX_VERBOSE_LOGGING: false
 
 jobs:
   main:
@@ -38,15 +38,14 @@ jobs:
         with:
           cmd1: npx nx-cloud record -- npx nx format:check
           cmd2: npx nx-cloud record -- npx nx workspace-lint
-          cmd3: npx nx-cloud record -- npm run ci:lint
-          cmd4: npx nx run-many --target=lint --all --parallel=3
-          # TODO: update this once project targets are converted
-          cmd5: npx nx run-many --target=test --projects=\"libs-*,nx-*\" --parallel=3
+          cmd3: npx nx run-many -t lint --parallel=3
+          cmd4: npx nx run-many -t test --parallel=3 --ci --maxWorkers=2
+          cmd5: npx nx run integration:integration --ci --maxWorkers=2
 
       # e2e tests for everything except the primary task runner
       - run: PUBLISHED_VERSION=999.9.9-e2e.0 npx nx run-many --target=run-e2e-tests-process --parallel=1
 
-      - name: Stop all running agents for this CI run
+      - name: Stop all running agents for Nx Run Group ${{ env.NX_RUN_GROUP }}
         # It's important that we always run this step, otherwise in the case of any failures in preceding non-Nx steps, the agents will keep running and waste billable minutes
         if: ${{ always() }}
         run: npx nx-cloud stop-all-agents
@@ -70,12 +69,92 @@ jobs:
 
       - run: npm run e2e-start-local-registry
 
-      - run: PUBLISHED_VERSION=999.9.9-e2e.0 npm run e2e-build-package-publish
+      - run: NX_VERBOSE_LOGGING=true PUBLISHED_VERSION=999.9.9-e2e.0 npm run e2e-build-package-publish
 
       - name: Start Nx Agent ${{ matrix.agent }}
         run: npx nx-cloud start-agent
         env:
-          NX_AGENT_NAME: ${{matrix.agent}}
+          NX_AGENT_NAME: ${{ matrix.agent }}
+
+  windows-main:
+    name: Nx Cloud - Windows Main Job
+    runs-on: windows-latest
+    env:
+      LERNA_OS_TYPE: windows
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Derive appropriate SHAs for base and head for `nx affected` commands
+        uses: nrwl/nx-set-shas@v3
+
+      - name: Set NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-windows" >> $GITHUB_ENV
+
+      - name: Log NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP is ${{ env.NX_RUN_GROUP }}"
+
+      - name: Install primary node version (see volta config in package.json) and dependencies
+        uses: ./.github/actions/install-node-and-dependencies
+
+      - name: Run parallel distributed unit and integration tests on windows
+        shell: bash
+        # From old maintainer regarding integration tests: "import is NOT TESTED in windows because pain and suffering"
+        run: |
+          pids=()
+          npx nx run integration:integration --ci --maxWorkers=2 --testPathIgnorePatterns=lerna-import.spec.ts &
+          pids+=($!)
+
+          # Ignored specs currently failing on windows
+          # TODO: investigate why
+          npx nx run-many -t test --parallel=3 --ci --maxWorkers=2 --testTimeout=45000 --testPathIgnorePatterns=create-symlink.spec.ts --testPathIgnorePatterns=get-two-factor-auth-required.spec.ts --testPathIgnorePatterns=version-lifecycle-scripts.spec.ts --testPathIgnorePatterns=version-allow-branch.spec.ts --testPathIgnorePatterns=version-message.spec.ts --testPathIgnorePatterns=version-conventional-commits.spec.ts --testPathIgnorePatterns=version-bump-prerelease.spec.ts --testPathIgnorePatterns=version-bump.spec.ts --testPathIgnorePatterns=version-git-hosted-siblings.spec.ts --testPathIgnorePatterns=version-command.spec.ts  --testPathIgnorePatterns=bootstrap-command.spec.ts --testPathIgnorePatterns=import-command.spec.ts --testPathIgnorePatterns=publish-command.spec.ts --testPathIgnorePatterns=publish-licenses.spec.ts --testPathIgnorePatterns=publish-lifecycle-scripts.spec.ts --testPathIgnorePatterns=publish-tagging.spec.ts --testPathIgnorePatterns=publish-relative-file-specifiers.spec.ts &
+
+          pids+=($!)
+
+          for pid in "${pids[@]}"; do
+            wait "$pid"
+          done
+
+      - name: Stop all running agents for Nx Run Group ${{ env.NX_RUN_GROUP }}
+        # It's important that we always run this step, otherwise in the case of any failures in preceding non-Nx steps, the agents will keep running and waste billable minutes
+        if: ${{ always() }}
+        run: npx nx-cloud stop-all-agents
+
+  windows-agents:
+    name: Nx Cloud - Windows Agent ${{ matrix.agent }}
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        agent: [1, 2, 3, 4]
+    env:
+      LERNA_OS_TYPE: windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-windows" >> $GITHUB_ENV
+
+      - name: Log NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP is ${{ env.NX_RUN_GROUP }}"
+
+      - name: Configure git metadata
+        run: |
+          git config --global user.email test@example.com
+          git config --global user.name "Tester McPerson"
+
+      - name: Install primary node version (see volta config in package.json) and dependencies
+        uses: ./.github/actions/install-node-and-dependencies
+
+      - name: Start Nx Agent ${{ matrix.agent }}
+        run: npx nx-cloud start-agent
+        env:
+          NX_AGENT_NAME: ${{ matrix.agent }}
+          NX_RUN_GROUP: ${{ env.NX_RUN_GROUP }}
 
   # Isolated e2e tests for the task runner which become too flaky if nested through further node child processes
   task-runner-e2e:
@@ -125,50 +204,3 @@ jobs:
         if: ${{ steps.check_affected.outputs.is_affected }}
         run: e2e/run/task-runner/src/run-tests.sh
         shell: bash
-
-  # TODO: migrate to cacheable/distributable targets
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        node: [14, "16.15.0", 18]
-        script: ["ci:test:unit", "ci:test:integration"]
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Configure git metadata
-        run: |
-          git config --global user.email test@example.com
-          git config --global user.name "Tester McPerson"
-
-      - name: Install node version ${{ matrix.node }} and dependencies
-        uses: ./.github/actions/install-node-and-dependencies
-        with:
-          node-version: ${{ matrix.node }}
-
-      - run: npm run ${{ matrix.script }}
-
-  # TODO: migrate to cacheable/distributable targets
-  windows:
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        script: ["ci:test:unit", "ci:test:integration"]
-        subset: [publish, non-publish]
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Configure git metadata
-        run: |
-          git config --global user.email test@example.com
-          git config --global user.name "Tester McPerson"
-
-      - name: Install primary node version (see volta config in package.json) and dependencies
-        uses: ./.github/actions/install-node-and-dependencies
-
-      - run: npm run ${{ matrix.script }}
-        env:
-          LERNA_CI_TYPE: ${{ matrix.subset }}
-          LERNA_OS_TYPE: windows

.github/workflows/other-node-versions.yml

@@ -0,0 +1,146 @@
+name: Other Node Versions
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Every day at 7:30am UTC
+    - cron: "30 7 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+  NX_CLOUD_DISTRIBUTED_EXECUTION: true
+  NX_BRANCH: ${{ github.event.number || github.ref_name }}
+  NX_VERBOSE_LOGGING: true
+
+jobs:
+  set-node-versions:
+    name: Set Node Versions
+    runs-on: ubuntu-latest
+    outputs:
+      node-versions: ${{ steps.node-matrix.outputs.node-versions }}
+    steps:
+      - name: Set node matrix
+        id: node-matrix
+        # These versions intentionally do not contain our primary major version because that has already been tested by ci.yml
+        # 19 is included so that we can test npm v9
+        #
+        # TODO: Investigate unit test failure on v19
+        # TODO: ^14.15.0 is currently failing e2e because of later JS usage in our e2e-utils, so we need to address that
+        run: |
+          NODE_VERSIONS_JSON_ARRAY=$(node -e "console.log(JSON.stringify(['18']));")
+          echo $NODE_VERSIONS_JSON_ARRAY
+          echo "node-versions=$NODE_VERSIONS_JSON_ARRAY" >> $GITHUB_OUTPUT
+
+  main:
+    name: Nx Cloud - Main Job - node-${{ matrix.node }}
+    needs: set-node-versions
+    runs-on: ubuntu-latest
+    strategy:
+      # Do not kill all versions of node just because one version failed
+      fail-fast: false
+      matrix:
+        node: ${{ fromJson(needs.set-node-versions.outputs.node-versions) }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Derive appropriate SHAs for base and head for `nx affected` commands
+        uses: nrwl/nx-set-shas@v3
+
+      - name: Set NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-node-${{ matrix.node }}" >> $GITHUB_ENV
+
+      - name: Log NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP is ${{ env.NX_RUN_GROUP }}"
+
+      - name: Install node v${{ matrix.node }} and dependencies
+        uses: ./.github/actions/install-node-and-dependencies
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Install pnpm
+        run: npm install -g pnpm
+        shell: bash
+
+      - run: pnpm --version
+        shell: bash
+
+      - name: Run parallel distributed tests on each node version
+        uses: jameshenry/parallel-bash-commands@v1
+        with:
+          cmd1: npx nx run-many -t test --parallel=3 --ci --maxWorkers=2
+          cmd2: npx nx run integration:integration --ci --maxWorkers=2
+
+      # e2e tests for everything except the primary task runner
+      - run: PUBLISHED_VERSION=999.9.9-e2e.0 npx nx run-many --target=run-e2e-tests-process --parallel=1
+
+      - name: Configure git metadata
+        run: |
+          git config --global user.email test@example.com
+          git config --global user.name "Tester McPerson"
+
+      - name: Prepare e2e tests for task-runner
+        run: npx nx prepare-for-e2e e2e-run-task-runner
+        shell: bash
+        env:
+          NX_CLOUD_DISTRIBUTED_EXECUTION: false
+
+      - name: Run e2e tests for task-runner
+        run: e2e/run/task-runner/src/run-tests.sh
+        shell: bash
+        env:
+          NX_CLOUD_DISTRIBUTED_EXECUTION: false
+
+      - name: Stop all running agents for Nx Run Group ${{ env.NX_RUN_GROUP }}
+        # It's important that we always run this step, otherwise in the case of any failures in preceding non-Nx steps, the agents will keep running and waste billable minutes
+        if: ${{ always() }}
+        run: npx nx-cloud stop-all-agents
+
+  agents:
+    name: Nx Cloud - Agent - node-${{ matrix.node }}-agent-${{ matrix.agent }}
+    needs: set-node-versions
+    runs-on: ubuntu-latest
+    strategy:
+      # Do not kill all versions of node just because one version failed
+      fail-fast: false
+      matrix:
+        node: ${{ fromJson(needs.set-node-versions.outputs.node-versions) }}
+        # Create 2 agents per node version
+        agent: [1, 2]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Configure git metadata
+        run: |
+          git config --global user.email test@example.com
+          git config --global user.name "Tester McPerson"
+
+      - name: Set NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-node-${{ matrix.node }}" >> $GITHUB_ENV
+
+      - name: Log NX_RUN_GROUP
+        shell: bash
+        run: echo "NX_RUN_GROUP is ${{ env.NX_RUN_GROUP }}"
+
+      - name: Install node v${{ matrix.node }} and dependencies
+        uses: ./.github/actions/install-node-and-dependencies
+        with:
+          node-version: ${{ matrix.node }}
+
+      - run: npm run e2e-start-local-registry
+
+      - run: NX_VERBOSE_LOGGING=true PUBLISHED_VERSION=999.9.9-e2e.0 npm run e2e-build-package-publish
+
+      - name: Start Nx Agent node-${{ matrix.node }}-agent-${{ matrix.agent }}
+        run: npx nx-cloud start-agent
+        env:
+          NX_AGENT_NAME: node-${{ matrix.node }}-agent-${{ matrix.agent }}
+          NX_RUN_GROUP: ${{ env.NX_RUN_GROUP }}

.npmrc

@@ -1,2 +1,5 @@
 # just in case a private registry is configured in ~/.npmrc
 registry = https://registry.npmjs.org/
+
+# TODO: remove this once esbuild 0.17 is officially supported in @nrwl/esbuild
+legacy-peer-deps = true

.prettierignore

@@ -1,5 +1,6 @@
-# Ignore json configuration files in fixtures, sometimes they are intentonally malformed
+# Ignore relevant files in fixtures, sometimes they are intentonally malformed
 **/__fixtures__/**/*.json
+**/__fixtures__/**/*.md
 
 dist
 tmp