Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #1576 from snyk/feat/test-iac-parallel-requests-cc…
…-594

Update snyk iac cli to issue parallel requests
  • Loading branch information
ipapast committed Jan 20, 2021
2 parents 8569305 + 544a793 commit ccb9baa
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 104 deletions.
1 change: 1 addition & 0 deletions package.json
Expand Up @@ -76,6 +76,7 @@
"needle": "2.5.0",
"open": "^7.0.3",
"os-name": "^3.0.0",
"promise-queue": "^2.2.5",
"proxy-agent": "^3.1.1",
"proxy-from-env": "^1.0.0",
"semver": "^6.0.0",
Expand Down
44 changes: 25 additions & 19 deletions src/lib/iac/detect-iac.ts
Expand Up @@ -21,6 +21,7 @@ import {
IllegalTerraformFileError,
} from '../errors/invalid-iac-file';
import { Options, TestOptions, IacFileInDirectory } from '../types';
import * as Queue from 'promise-queue';

const debug = debugLib('snyk-detect-iac');

Expand Down Expand Up @@ -90,14 +91,17 @@ async function getDirectoryFiles(
root: string,
options: { maxDepth?: number } = {},
) {
const iacFiles: IacFileInDirectory[] = [];
const dirPath = pathLib.resolve(root, '.');
const supportedExtensions = new Set(Object.keys(projectTypeByFileType));

const directoryPaths = makeDirectoryIterator(dirPath, {
maxDepth: options.maxDepth,
});

const iacFiles: IacFileInDirectory[] = [];
const maxConcurrent = 25;
const queue = new Queue(maxConcurrent);

for (const filePath of directoryPaths) {
const fileType = pathLib
.extname(filePath)
Expand All @@ -106,27 +110,29 @@ async function getDirectoryFiles(
if (!fileType || !supportedExtensions.has(fileType)) {
continue;
}

try {
const projectType = await getProjectTypeForIacFile(filePath);
iacFiles.push({
filePath,
projectType,
fileType,
});
} catch (err) {
iacFiles.push({
filePath,
fileType,
failureReason:
err instanceof CustomError ? err.userMessage : 'Unhandled Error',
});
}
iacFiles.push(
queue.add(async () => {
try {
const projectType = await getProjectTypeForIacFile(filePath);
return {
filePath,
projectType,
fileType,
};
} catch (err) {
return {
filePath,
fileType,
failureReason:
err instanceof CustomError ? err.userMessage : 'Unhandled Error',
};
}
}),
);
}

if (iacFiles.length === 0) {
throw IacDirectoryWithoutAnyIacFileError();
}

return iacFiles;
return Promise.all(iacFiles);
}
13 changes: 10 additions & 3 deletions src/lib/request/request.ts
Expand Up @@ -7,10 +7,11 @@ import * as config from '../config';
import { getProxyForUrl } from 'proxy-from-env';
import * as ProxyAgent from 'proxy-agent';
import * as analytics from '../analytics';
import { Agent } from 'http';
import { Global } from '../../cli/args';
import { Payload } from './types';
import { getVersion } from '../version';
import * as https from 'https';
import * as http from 'http';

const debug = debugModule('snyk:req');
const snykDebug = debugModule('snyk');
Expand Down Expand Up @@ -94,25 +95,31 @@ export = function makeRequest(
delete payload.qs;
}

const agent =
parsedUrl.protocol === 'http:'
? new http.Agent({ keepAlive: true })
: new https.Agent({ keepAlive: true });
const options: needle.NeedleOptions = {
json: payload.json,
headers: payload.headers,
timeout: payload.timeout,
// eslint-disable-next-line @typescript-eslint/camelcase
follow_max: 5,
family: payload.family,
agent,
};

const proxyUri = getProxyForUrl(url);
if (proxyUri) {
snykDebug('using proxy:', proxyUri);
options.agent = (new ProxyAgent(proxyUri) as unknown) as Agent;
// proxyAgent type is an EventEmitter and not an http Agent
options.agent = (new ProxyAgent(proxyUri) as unknown) as http.Agent;
} else {
snykDebug('not using proxy');
}

if (global.ignoreUnknownCA) {
debug('Using insecure mode (ignore unkown certificate authority)');
debug('Using insecure mode (ignore unknown certificate authority)');
options.rejectUnauthorized = false;
}

Expand Down
166 changes: 92 additions & 74 deletions src/lib/snyk-test/run-test.ts
@@ -1,58 +1,56 @@
import * as fs from 'fs';
import * as _ from 'lodash';
import * as path from 'path';
import * as pathUtil from 'path';
import * as debugModule from 'debug';
import chalk from 'chalk';
import * as pathUtil from 'path';
import { parsePackageString as moduleToObject } from 'snyk-module';
import * as depGraphLib from '@snyk/dep-graph';
import { IacScan } from './payload-schema';
import * as Queue from 'promise-queue';

import {
TestResult,
DockerIssue,
AffectedPackages,
AnnotatedIssue,
TestDepGraphResponse,
convertTestDepGraphResultToLegacy,
DockerIssue,
LegacyVulnApiResult,
TestDependenciesResponse,
AffectedPackages,
TestDepGraphResponse,
TestResult,
} from './legacy';
import { IacTestResponse } from './iac-test-result';
import {
AuthFailedError,
InternalServerError,
NoSupportedManifestsFoundError,
DockerImageNotFoundError,
FailedToGetVulnerabilitiesError,
FailedToGetVulnsFromUnavailableResource,
FailedToRunTestError,
InternalServerError,
NoSupportedManifestsFoundError,
UnsupportedFeatureFlagError,
DockerImageNotFoundError,
} from '../errors';
import * as snyk from '../';
import { isCI } from '../is-ci';
import * as common from './common';
import * as config from '../config';
import * as analytics from '../analytics';
import { maybePrintDepTree, maybePrintDepGraph } from '../print-deps';
import { GitTarget, ContainerTarget } from '../project-metadata/types';
import { maybePrintDepGraph, maybePrintDepTree } from '../print-deps';
import { ContainerTarget, GitTarget } from '../project-metadata/types';
import * as projectMetadata from '../project-metadata';
import {
DepTree,
Options,
TestOptions,
SupportedProjectTypes,
PolicyOptions,
SupportedProjectTypes,
TestOptions,
} from '../types';
import { pruneGraph } from '../prune';
import { getDepsFromPlugin } from '../plugins/get-deps-from-plugin';
import {
ScannedProjectCustom,
MultiProjectResultCustom,
ScannedProjectCustom,
} from '../plugins/get-multi-plugin-result';

import request = require('../request');
import spinner = require('../spinner');
import { extractPackageManager } from '../plugins/extract-package-manager';
import { getExtraProjectCount } from '../plugins/get-extra-project-count';
import { serializeCallGraphWithMetrics } from '../reachable-vulns';
Expand All @@ -73,6 +71,8 @@ import { getEcosystem } from '../ecosystems';
import { Issue } from '../ecosystems/types';
import { assembleEcosystemPayloads } from './assemble-payloads';
import { NonExistingPackageError } from '../errors/non-existing-package-error';
import request = require('../request');
import spinner = require('../spinner');

const debug = debugModule('snyk:run-test');

Expand Down Expand Up @@ -225,72 +225,90 @@ async function sendAndParseResults(
root: string,
options: Options & TestOptions,
): Promise<TestResult[]> {
// Note for the IaC Test Flow:
// There is a RATE_LIMIT setup for network requests within a time period.
// To support this limit and avoid getting back 502 errors from registry,
// we introduced a concurrent requests limit of 25. With that, we will only process MAX 25 requests at the same time.
// In the future, we would probably want to introduce a RATE_LIMIT specific for IaC

if (options.iac) {
const maxConcurrent = 25;
const queue = new Queue(maxConcurrent);
const iacResults: Promise<TestResult>[] = [];

await spinner.clear<void>(spinnerLbl)();
await spinner(spinnerLbl);
for (const payload of payloads) {
iacResults.push(
queue.add(async () => {
const iacScan: IacScan = payload.body as IacScan;
analytics.add('iac type', !!iacScan.type);
const res = (await sendTestPayload(payload)) as IacTestResponse;

const projectName =
iacScan.projectNameOverride || iacScan.originalProjectName;
return await parseIacTestResult(
res,
iacScan.targetFile,
projectName,
options.severityThreshold,
);
}),
);
}
return Promise.all(iacResults);
}

const results: TestResult[] = [];
for (const payload of payloads) {
await spinner.clear<void>(spinnerLbl)();
await spinner(spinnerLbl);
if (options.iac) {
const iacScan: IacScan = payload.body as IacScan;
analytics.add('iac type', !!iacScan.type);
const res = (await sendTestPayload(payload)) as IacTestResponse;

const projectName =
iacScan.projectNameOverride || iacScan.originalProjectName;
const result = await parseIacTestResult(
res,
iacScan.targetFile,
projectName,
options.severityThreshold,
);
results.push(result);
} else {
/** sendTestPayload() deletes the request.body from the payload once completed. */
const payloadCopy = Object.assign({}, payload);
const res = await sendTestPayload(payload);
const {
depGraph,
payloadPolicy,
pkgManager,
targetFile,
projectName,
foundProjectCount,
displayTargetFile,
dockerfilePackages,
platform,
} = prepareResponseForParsing(
payloadCopy,
res as TestDependenciesResponse,
options,
);
/** sendTestPayload() deletes the request.body from the payload once completed. */
const payloadCopy = Object.assign({}, payload);
const res = await sendTestPayload(payload);
const {
depGraph,
payloadPolicy,
pkgManager,
targetFile,
projectName,
foundProjectCount,
displayTargetFile,
dockerfilePackages,
platform,
} = prepareResponseForParsing(
payloadCopy,
res as TestDependenciesResponse,
options,
);

const ecosystem = getEcosystem(options);
if (ecosystem && options['print-deps']) {
await spinner.clear<void>(spinnerLbl)();
await maybePrintDepGraph(options, depGraph);
}
const ecosystem = getEcosystem(options);
if (ecosystem && options['print-deps']) {
await spinner.clear<void>(spinnerLbl)();
await maybePrintDepGraph(options, depGraph);
}

const legacyRes = convertIssuesToAffectedPkgs(res);
const legacyRes = convertIssuesToAffectedPkgs(res);

const result = await parseRes(
depGraph,
pkgManager,
legacyRes as LegacyVulnApiResult,
options,
payload,
payloadPolicy,
root,
dockerfilePackages,
);
const result = await parseRes(
depGraph,
pkgManager,
legacyRes as LegacyVulnApiResult,
options,
payload,
payloadPolicy,
root,
dockerfilePackages,
);

results.push({
...result,
targetFile,
projectName,
foundProjectCount,
displayTargetFile,
platform,
});
}
results.push({
...result,
targetFile,
projectName,
foundProjectCount,
displayTargetFile,
platform,
});
}
return results;
}
Expand Down

0 comments on commit ccb9baa

Please sign in to comment.