Merge pull request #1521 from snyk/feature/iac-detection-depth-CC-434

feat: Implement --detection-depth flag in iac command

Author: aron

help/commands-docs/iac-examples.md

@@ -7,3 +7,6 @@
 
 - `Test terraform file`:
   \$ snyk iac test /path/to/terraform_file.tf
+
+- `Test matching files in a directory`:
+  \$ snyk iac test /path/to/directory

help/commands-docs/iac.md

@@ -17,6 +17,14 @@ Find security issues in your Infrastructure as Code files.
 
 ## OPTIONS
 
+- `--detection-depth`=<DEPTH>:
+  (only in `test` command)  
+  Indicate the maximum depth of sub-directories to search. <DEPTH> must be a number.
+
+  Default: No Limit  
+  Example: `--detection-depth=3`  
+  Will limit search to provided directory (or current directory if no <PATH> provided) plus two levels of subdirectories.
+
 - `--severity-threshold`=low|medium|high:
   Only report vulnerabilities of provided level or higher.
 

package.json

@@ -69,7 +69,6 @@
     "configstore": "^5.0.1",
     "debug": "^4.1.1",
     "diff": "^4.0.1",
-    "glob": "^7.1.3",
     "graphlib": "^2.1.8",
     "inquirer": "^7.3.3",
     "lodash": "^4.17.20",

src/cli/args.ts

@@ -68,7 +68,7 @@ export interface ArgsOptions {
   // (see the snyk-mvn-plugin or snyk-gradle-plugin)
   _doubleDashArgs: string[];
   _: MethodArgs;
-  [key: string]: boolean | string | MethodArgs | string[]; // The two last types are for compatibility only
+  [key: string]: boolean | string | number | MethodArgs | string[]; // The two last types are for compatibility only
 }
 
 export function args(rawArgv: string[]): Args {
@@ -227,6 +227,10 @@ export function args(rawArgv: string[]): Args {
     }
   }
 
+  if (argv.detectionDepth !== undefined) {
+    argv.detectionDepth = Number(argv.detectionDepth);
+  }
+
   if (argv.skipUnresolved !== undefined) {
     if (argv.skipUnresolved === 'false') {
       argv.allowMissing = false;

src/cli/index.ts

@@ -40,6 +40,7 @@ import {
   SupportedUserReachableFacingCliArgs,
 } from '../lib/types';
 import { SarifFileOutputEmptyError } from '../lib/errors/empty-sarif-output-error';
+import { InvalidDetectionDepthValue } from '../lib/errors/invalid-detection-depth-value';
 
 const debug = Debug('snyk');
 const EXIT_CODES = {
@@ -262,6 +263,14 @@ async function main() {
       throw new FileFlagBadInputError();
     }
 
+    if (
+      typeof args.options.detectionDepth !== 'undefined' &&
+      (args.options.detectionDepth <= 0 ||
+        Number.isNaN(args.options.detectionDepth))
+    ) {
+      throw new InvalidDetectionDepthValue();
+    }
+
     validateUnsupportedSarifCombinations(args);
 
     validateOutputFile(args.options, 'json', new JsonFileOutputBadInputError());

src/lib/detect.ts

@@ -153,7 +153,7 @@ export function localFileSuppliedButNotFound(root, file) {
   );
 }
 
-export function isLocalFolder(root) {
+export function isLocalFolder(root: string) {
   try {
     return fs.lstatSync(root).isDirectory();
   } catch (e) {

src/lib/errors/invalid-detection-depth-value.ts

@@ -0,0 +1,10 @@
+import { CustomError } from './custom-error';
+
+export class InvalidDetectionDepthValue extends CustomError {
+  constructor() {
+    const msg = `Unsupported value for --detection-depth flag. Expected a positive integer.`;
+    super(msg);
+    this.code = 422;
+    this.userMessage = msg;
+  }
+}

src/lib/iac/detect-iac.ts

@@ -1,7 +1,6 @@
 import * as fs from 'fs';
 import * as pathLib from 'path';
 import * as debugLib from 'debug';
-import * as glob from 'glob';
 import { isLocalFolder, localFileSuppliedButNotFound } from '../detect';
 import { CustomError } from '../errors';
 import { validateK8sFile, makeValidateTerraformRequest } from './iac-parser';
@@ -10,6 +9,7 @@ import {
   IacProjectType,
   IacFileTypes,
 } from './constants';
+import { makeDirectoryIterator } from './makeDirectoryIterator';
 import {
   SupportLocalFileOnlyIacError,
   UnsupportedOptionFileIacError,
@@ -38,7 +38,9 @@ export async function getProjectType(
     // scanning the projects - we need save the files we need to scan on the options
     // so we could create assembly payloads for the relevant files.
     // We are sending it as a `Multi IaC` project - and later assign the relevant type for each project
-    const directoryFiles = await getDirectoryFiles(root);
+    const directoryFiles = await getDirectoryFiles(root, {
+      maxDepth: options.detectionDepth,
+    });
     options.iacDirFiles = directoryFiles;
     return IacProjectType.MULTI_IAC;
   }
@@ -84,33 +86,41 @@ async function getProjectTypeForIacFile(filePath: string) {
   return projectType;
 }
 
-async function getDirectoryFiles(root: string) {
+async function getDirectoryFiles(
+  root: string,
+  options: { maxDepth?: number } = {},
+) {
   const iacFiles: IacFileInDirectory[] = [];
   const dirPath = pathLib.resolve(root, '.');
-  const files = glob.sync(
-    pathLib.join(dirPath, '/**/**/*.+(json|yaml|yml|tf)'),
-  );
+  const supportedExtensions = new Set(Object.keys(projectTypeByFileType));
+
+  const directoryPaths = makeDirectoryIterator(dirPath, {
+    maxDepth: options.maxDepth,
+  });
 
-  for (const fileName of files) {
-    const ext = pathLib.extname(fileName).substr(1);
-    if (Object.keys(projectTypeByFileType).includes(ext)) {
-      const filePath = pathLib.resolve(root, fileName);
+  for (const filePath of directoryPaths) {
+    const fileType = pathLib
+      .extname(filePath)
+      .substr(1)
+      .toLowerCase() as IacFileTypes;
+    if (!fileType || !supportedExtensions.has(fileType)) {
+      continue;
+    }
 
-      await getProjectTypeForIacFile(filePath)
-        .then((projectType) => {
-          iacFiles.push({
-            filePath,
-            projectType,
-            fileType: ext as IacFileTypes,
-          });
-        })
-        .catch((err: CustomError) => {
-          iacFiles.push({
-            filePath,
-            fileType: ext as IacFileTypes,
-            failureReason: err.userMessage,
-          });
-        });
+    try {
+      const projectType = await getProjectTypeForIacFile(filePath);
+      iacFiles.push({
+        filePath,
+        projectType,
+        fileType,
+      });
+    } catch (err) {
+      iacFiles.push({
+        filePath,
+        fileType,
+        failureReason:
+          err instanceof CustomError ? err.userMessage : 'Unhandled Error',
+      });
     }
   }
 

src/lib/iac/makeDirectoryIterator.ts

@@ -0,0 +1,54 @@
+import * as fs from 'fs';
+import { resolve } from 'path';
+
+/**
+ * Walk the provided directory tree, depth first, yielding the matched filename
+ * to the caller. An optional `maxDepth` argument can be provided to limit how
+ * deep in the file tree the search will go.
+ */
+export function* makeDirectoryIterator(
+  root: string,
+  options: { maxDepth?: number; includeDotfiles?: boolean } = {},
+) {
+  if (!isDirectory(root)) {
+    throw new Error(`Path "${root}" is not a directory`);
+  }
+
+  // Internal function to allow us to track the recursion depth.
+  function* walk(dirpath: string, currentDepth: number) {
+    const filenames = fs.readdirSync(dirpath);
+    for (const filename of filenames) {
+      // NOTE: We filter filenames that start with a period to maintain
+      // backwards compatibility with the original implementation that used the
+      // "glob" package.
+      if (!options.includeDotfiles && filename.startsWith('.')) {
+        continue;
+      }
+
+      const resolved = resolve(dirpath, filename);
+      if (isDirectory(resolved)) {
+        // Skip this directory if max depth has been reached.
+        if (options.maxDepth === currentDepth) {
+          continue;
+        }
+
+        yield* walk(resolved, currentDepth + 1);
+      } else {
+        yield resolved;
+      }
+    }
+  }
+
+  yield* walk(root, 1);
+}
+
+// NOTE: We use isDirectory here instead of isLocalFolder() in order to
+// follow symlinks and match the original glob() implementation.
+function isDirectory(path: string): boolean {
+  try {
+    // statSync will resolve symlinks, lstatSync will not.
+    return fs.statSync(path).isDirectory();
+  } catch (e) {
+    return false;
+  }
+}

test/fixtures/iac/depth_detection/.hidden.tf

@@ -0,0 +1,54 @@
+import { makeDirectoryIterator } from '../src/lib/iac/makeDirectoryIterator';
+import * as path from 'path';
+
+describe('makeDirectoryIterator', () => {
+  const fixturePath = path.join(__dirname, 'fixtures/iac/depth_detection');
+
+  it('iterates over all files in the directory tree', () => {
+    const it = makeDirectoryIterator(fixturePath);
+    const result = Array.from(it);
+
+    expect(result).toEqual([
+      expect.stringContaining('one.tf'),
+      expect.stringContaining('five.tf'),
+      expect.stringContaining('six.tf'),
+      expect.stringContaining('four.tf'),
+      expect.stringContaining('three.tf'),
+      expect.stringContaining('two.tf'),
+      expect.stringContaining('root.tf'),
+    ]);
+  });
+
+  it('includes dotfiles when the includeDotfiles flag is provided', () => {
+    const it = makeDirectoryIterator(fixturePath, { includeDotfiles: true });
+    const result = Array.from(it);
+
+    expect(result).toEqual([
+      expect.stringContaining('hidden.tf'),
+      expect.stringContaining('.hidden.tf'),
+      expect.stringContaining('one.tf'),
+      expect.stringContaining('five.tf'),
+      expect.stringContaining('six.tf'),
+      expect.stringContaining('four.tf'),
+      expect.stringContaining('three.tf'),
+      expect.stringContaining('two.tf'),
+      expect.stringContaining('root.tf'),
+    ]);
+  });
+
+  it('limits the subdirectory depth when maxDepth option is provided', () => {
+    const it = makeDirectoryIterator(fixturePath, { maxDepth: 3 });
+    const result = Array.from(it);
+
+    expect(result).toEqual([
+      expect.stringContaining('one.tf'),
+      expect.stringContaining('two.tf'),
+      expect.stringContaining('root.tf'),
+    ]);
+  });
+
+  it('throws an error if the path provided is not a directory', () => {
+    const it = makeDirectoryIterator('missing_path');
+    expect(() => Array.from(it)).toThrowError();
+  });
+});