Merge pull request #226 from snyk/feat/normalize-deps

feat: normalize deps
snyk · Sep 23, 2022 · a4eb261 · a4eb261
2 parents 3ea5ec1 + cad5070
commit a4eb261
Show file tree

Hide file tree

Showing 12 changed files with 496 additions and 51 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -21,6 +21,7 @@ windows_defaults: &windows_defaults
     npm_config_loglevel: silent
   executor:
     name: win/default
+    size: large
 
 test_matrix_unix: &test_matrix_unix
   matrix:

diff --git a/lib/graph.ts b/lib/graph.ts
@@ -1,5 +1,7 @@
 import { DepGraphBuilder, PkgManager } from '@snyk/dep-graph';
 
+import type { CoordinateMap } from './types';
+
 export interface SnykGraph {
   [id: string]: {
     name: string;
@@ -17,6 +19,7 @@ export async function buildGraph(
   snykGraph: SnykGraph,
   projectName: string,
   projectVersion: string,
+  coordinateMap?: CoordinateMap,
 ) {
   const pkgManager: PkgManager = { name: 'gradle' };
   const isEmptyGraph = !snykGraph || Object.keys(snykGraph).length === 0;
@@ -38,10 +41,22 @@ export async function buildGraph(
   while (queue.length > 0) {
     const item = queue.shift();
     if (!item) continue;
-    const { id, parentId } = item;
+    let { id, parentId } = item;
     const node = snykGraph[id];
     if (!node) continue;
-    const { name = 'unknown', version = 'unknown' } = node;
+    let { name = 'unknown', version = 'unknown' } = node;
+
+    if (coordinateMap) {
+      if (coordinateMap[id]) {
+        id = coordinateMap[id];
+        const [newName, newVersion] = id.split('@');
+        name = newName;
+        version = newVersion;
+      }
+      if (coordinateMap[parentId]) {
+        parentId = coordinateMap[parentId];
+      }
+    }
     if (visited.includes(id)) {
       const prunedId = id + ':pruned';
       depGraphBuilder.addPkgNode({ name, version }, prunedId, {

diff --git a/lib/index.ts b/lib/index.ts
@@ -3,6 +3,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as subProcess from './sub-process';
 import * as tmp from 'tmp';
+import * as pMap from 'p-map';
 import { MissingSubProjectError } from './errors';
 import * as chalk from 'chalk';
 import { DepGraph } from '@snyk/dep-graph';
@@ -11,6 +12,12 @@ import * as javaCallGraphBuilder from '@snyk/java-call-graph-builder';
 import { getGradleAttributesPretty } from './gradle-attributes-pretty';
 import debugModule = require('debug');
 import { buildGraph, SnykGraph } from './graph';
+import type {
+  CoordinateMap,
+  PomCoords,
+  Sha1Map,
+  SnykHttpClient,
+} from './types';
 
 type ScannedProject = legacyCommon.ScannedProject;
 type CallGraph = legacyCommon.CallGraph;
@@ -67,6 +74,7 @@ export interface GradleInspectOptions {
   reachableVulns?: boolean;
   callGraphBuilderTimeout?: number;
   initScript?: string;
+  gradleNormalizeDeps?: boolean;
 }
 
 type Options = api.InspectOptions & GradleInspectOptions;
@@ -78,18 +86,21 @@ export async function inspect(
   root: string,
   targetFile: string,
   options?: api.SingleSubprojectInspectOptions & GradleInspectOptions,
+  snykHttpClient?: SnykHttpClient,
 ): Promise<api.SinglePackageResult>;
 export async function inspect(
   root: string,
   targetFile: string,
   options: api.MultiSubprojectInspectOptions & GradleInspectOptions,
+  snykHttpClient?: SnykHttpClient,
 ): Promise<api.MultiProjectResult>;
 
 // General implementation. The result type depends on the runtime type of `options`.
 export async function inspect(
   root: string,
   targetFile: string,
   options?: Options,
+  snykHttpClient?: SnykHttpClient,
 ): Promise<api.InspectResult> {
   debugLog(
     'Gradle inspect called with: ' +
@@ -100,6 +111,7 @@ export async function inspect(
         subProject: (options as any)?.subProject,
       }),
   );
+
   if (!options) {
     options = { dev: false };
   }
@@ -157,6 +169,7 @@ export async function inspect(
       root,
       targetFile,
       options,
+      snykHttpClient,
     );
     plugin.meta = plugin.meta || {};
     return {
@@ -169,6 +182,7 @@ export async function inspect(
     root,
     targetFile,
     options,
+    snykHttpClient,
     subProject,
   );
   if (depGraphAndDepRootNames.allSubProjectNames) {
@@ -206,6 +220,7 @@ export interface JsonDepsScriptResult {
   projects: ProjectsDict;
   allSubProjectNames: string[];
   versionBuildInfo?: VersionBuildInfo;
+  sha1Map?: Sha1Map;
 }
 
 interface ProjectsDict {
@@ -255,14 +270,20 @@ async function getAllDepsOneProject(
   root: string,
   targetFile: string,
   options: Options,
+  snykHttpClient: SnykHttpClient,
   subProject?: string,
 ): Promise<{
   depGraph: DepGraph;
   allSubProjectNames: string[];
   gradleProjectName: string;
   versionBuildInfo: VersionBuildInfo;
 }> {
-  const allProjectDeps = await getAllDeps(root, targetFile, options);
+  const allProjectDeps = await getAllDeps(
+    root,
+    targetFile,
+    options,
+    snykHttpClient,
+  );
   const allSubProjectNames = allProjectDeps.allSubProjectNames;
 
   return subProject
@@ -322,8 +343,14 @@ async function getAllDepsAllProjects(
   root: string,
   targetFile: string,
   options: Options,
+  snykHttpClient: SnykHttpClient,
 ): Promise<ScannedProject[]> {
-  const allProjectDeps = await getAllDeps(root, targetFile, options);
+  const allProjectDeps = await getAllDeps(
+    root,
+    targetFile,
+    options,
+    snykHttpClient,
+  );
   return Object.keys(allProjectDeps.projects).map((projectId) => {
     const { defaultProject } = allProjectDeps;
     const gradleProjectName =
@@ -486,10 +513,47 @@ async function getAllDepsWithPlugin(
   return extractedJSON;
 }
 
+export async function getMavenPackageInfo(
+  sha1: string,
+  depCoords: Partial<PomCoords>,
+  snykHttpClient: SnykHttpClient,
+): Promise<string> {
+  const { res, body } = await snykHttpClient({
+    method: 'get',
+    path: `/maven/coordinates/sha1/${sha1}`,
+    qs: depCoords,
+  });
+  if (res?.statusCode >= 400 || !body || !body.ok || body.code >= 400) {
+    debugLog(
+      body.message ||
+        `Failed to resolve ${JSON.stringify(depCoords)} using sha1 '${sha1}.`,
+    );
+  }
+  const {
+    groupId = depCoords.groupId || 'unknown',
+    artifactId = depCoords.artifactId || 'unknown',
+    version = depCoords.version || 'unknown',
+  } = body.coordinate || {};
+  return `${groupId}:${artifactId}@${version}`;
+}
+
+function splitCoordinate(coordinate: string): Partial<PomCoords> {
+  const coordTest = /^[\w.-]+:[\w.-]+@[\w.-]+$/.test(coordinate);
+  if (!coordTest) return {};
+  const [groupId, artifactId, version] = coordinate.split(/:|@/);
+  const pomCoord: Partial<PomCoords> = {};
+  pomCoord.artifactId = artifactId;
+  pomCoord.groupId = groupId;
+  pomCoord.version = version;
+
+  return pomCoord;
+}
+
 async function getAllDeps(
   root: string,
   targetFile: string,
   options: Options,
+  snykHttpClient: SnykHttpClient,
 ): Promise<JsonDepsScriptResult> {
   const command = getCommand(root, targetFile);
   const gradleVersion = await getGradleVersion(root, command);
@@ -503,7 +567,32 @@ async function getAllDeps(
     if (versionBuildInfo) {
       extractedJSON.versionBuildInfo = versionBuildInfo;
     }
-    return await processProjectsInExtractedJSON(root, extractedJSON);
+    const coordinateMap: CoordinateMap = {};
+    if (extractedJSON.sha1Map) {
+      const getCoordinateFromHash = async (hash: string): Promise<void> => {
+        const originalCoordinate = extractedJSON.sha1Map[hash];
+        const depCoord = splitCoordinate(originalCoordinate);
+        try {
+          const coordinate = await getMavenPackageInfo(
+            hash,
+            depCoord,
+            snykHttpClient,
+          );
+          coordinateMap[originalCoordinate] = coordinate;
+        } catch (err) {
+          debugLog(err);
+        }
+      };
+
+      await pMap(Object.keys(extractedJSON.sha1Map), getCoordinateFromHash, {
+        concurrency: 100,
+      });
+    }
+    return await processProjectsInExtractedJSON(
+      root,
+      extractedJSON,
+      coordinateMap,
+    );
   } catch (err) {
     const error: Error = err;
     const gradleErrorMarkers = /^\s*>\s.*$/;
@@ -589,6 +678,7 @@ ${chalk.red.bold(mainErrorMessage)}`;
 export async function processProjectsInExtractedJSON(
   root: string,
   extractedJSON: JsonDepsScriptResult,
+  coordinateMap?: CoordinateMap,
 ) {
   for (const projectId in extractedJSON.projects) {
     const { defaultProject } = extractedJSON;
@@ -614,6 +704,7 @@ export async function processProjectsInExtractedJSON(
       snykGraph,
       projectName,
       projectVersion,
+      coordinateMap,
     );
     // this property usage ends here
     delete extractedJSON.projects[projectId].snykGraph;
@@ -666,7 +757,12 @@ function buildArgs(
   options: Options,
 ) {
   const args: string[] = [];
-  args.push('snykResolvedDepsJson', '-q');
+  const taskName = options.gradleNormalizeDeps
+    ? 'snykNormalizedResolvedDepsJson'
+    : 'snykResolvedDepsJson';
+
+  args.push(taskName, '-q');
+
   if (targetFile) {
     if (!fs.existsSync(path.resolve(root, targetFile))) {
       throw new Error('File not found: "' + targetFile + '"');
@@ -772,4 +868,6 @@ export const exportsForTests = {
   getVersionBuildInfo,
   toCamelCase,
   getGradleAttributesPretty,
+  splitCoordinate,
+  getMavenPackageInfo,
 };