feat: Implement --detection-depth flag in iac module

Currently we look for IaC files within all subdirectories. This commit
refactors the detection code to use direct directory traversal function
that takes a `maxDepth` argument which will bail early once the max
depth has been reached.

This removes the dependency on the glob module, brings the file lookup
inline with how we do the lookup for manifest files and offers a slight
performance gain over glob.

To maintain backwards compatibility with the existing functionality
this implementation will traverse all directories by default and ignore
dotfiles (files starting with a period).

Author: aron

src/lib/iac/detect-iac.ts

@@ -1,7 +1,6 @@
 import * as fs from 'fs';
 import * as pathLib from 'path';
 import * as debugLib from 'debug';
-import * as glob from 'glob';
 import { isLocalFolder, localFileSuppliedButNotFound } from '../detect';
 import { CustomError } from '../errors';
 import { validateK8sFile, makeValidateTerraformRequest } from './iac-parser';
@@ -10,6 +9,7 @@ import {
   IacProjectType,
   IacFileTypes,
 } from './constants';
+import { makeDirectoryIterator } from './makeDirectoryIterator';
 import {
   SupportLocalFileOnlyIacError,
   UnsupportedOptionFileIacError,
@@ -38,7 +38,9 @@ export async function getProjectType(
     // scanning the projects - we need save the files we need to scan on the options
     // so we could create assembly payloads for the relevant files.
     // We are sending it as a `Multi IaC` project - and later assign the relevant type for each project
-    const directoryFiles = await getDirectoryFiles(root);
+    const directoryFiles = await getDirectoryFiles(root, {
+      maxDepth: options.detectionDepth,
+    });
     options.iacDirFiles = directoryFiles;
     return IacProjectType.MULTI_IAC;
   }
@@ -84,33 +86,38 @@ async function getProjectTypeForIacFile(filePath: string) {
   return projectType;
 }
 
-async function getDirectoryFiles(root: string) {
+async function getDirectoryFiles(
+  root: string,
+  options: { maxDepth?: number } = {},
+) {
   const iacFiles: IacFileInDirectory[] = [];
   const dirPath = pathLib.resolve(root, '.');
-  const files = glob.sync(
-    pathLib.join(dirPath, '/**/**/*.+(json|yaml|yml|tf)'),
-  );
+  const supportedExtensions = new Set(Object.keys(projectTypeByFileType));
+
+  const directoryPaths = makeDirectoryIterator(dirPath, {
+    maxDepth: options.maxDepth,
+  });
 
-  for (const fileName of files) {
-    const ext = pathLib.extname(fileName).substr(1);
-    if (Object.keys(projectTypeByFileType).includes(ext)) {
-      const filePath = pathLib.resolve(root, fileName);
+  for (const filePath of directoryPaths) {
+    const fileType = pathLib.extname(filePath).substr(1) as IacFileTypes;
+    if (!fileType || !supportedExtensions.has(fileType)) {
+      continue;
+    }
 
-      await getProjectTypeForIacFile(filePath)
-        .then((projectType) => {
-          iacFiles.push({
-            filePath,
-            projectType,
-            fileType: ext as IacFileTypes,
-          });
-        })
-        .catch((err: CustomError) => {
-          iacFiles.push({
-            filePath,
-            fileType: ext as IacFileTypes,
-            failureReason: err.userMessage,
-          });
-        });
+    try {
+      const projectType = await getProjectTypeForIacFile(filePath);
+      iacFiles.push({
+        filePath,
+        projectType,
+        fileType,
+      });
+    } catch (err) {
+      iacFiles.push({
+        filePath,
+        fileType,
+        failureReason:
+          err instanceof CustomError ? err.userMessage : 'Unhandled Error',
+      });
     }
   }
 

src/lib/iac/makeDirectoryIterator.ts

@@ -0,0 +1,54 @@
+import * as fs from 'fs';
+import { resolve } from 'path';
+
+/**
+ * Walk the provided directory tree, depth first, yielding the matched filename
+ * to the caller. An optional `maxDepth` argument can be provided to limit how
+ * deep in the file tree the search will go.
+ */
+export function* makeDirectoryIterator(
+  root: string,
+  options: { maxDepth?: number; includeDotfiles?: boolean } = {},
+) {
+  if (!isDirectory(root)) {
+    throw new Error(`Path "${root}" is not a directory`);
+  }
+
+  // Internal function to allow us to track the recursion depth.
+  function* walk(dirpath: string, currentDepth: number) {
+    const filenames = fs.readdirSync(dirpath);
+    for (const filename of filenames) {
+      // NOTE: We filter filenames that start with a period to maintain
+      // backwards compatibility with the original implementation that used the
+      // "glob" package.
+      if (!options.includeDotfiles && filename.startsWith('.')) {
+        continue;
+      }
+
+      const resolved = resolve(dirpath, filename);
+      if (isDirectory(resolved)) {
+        // Skip this directory if max depth has been reached.
+        if (options.maxDepth === currentDepth) {
+          continue;
+        }
+
+        yield* walk(resolved, currentDepth + 1);
+      } else {
+        yield resolved;
+      }
+    }
+  }
+
+  yield* walk(root, 1);
+}
+
+// NOTE: We use isDirectory here instead of isLocalFolder() in order to
+// follow symlinks and match the original glob() implementation.
+function isDirectory(path: string): boolean {
+  try {
+    // statSync will resolve symlinks, lstatSync will not.
+    return fs.statSync(path).isDirectory();
+  } catch (e) {
+    return false;
+  }
+}