Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
chore: init support async test with polling
Initial support for async test of unmanaged ecosystems e.g. c/c++
adding polling to resolve and test fileSignatures fact.

The pr that will introduce the active usage of it will come later on.

These changes comes from #2025

Using chore as prefix, because at the moment it's not bringing any
effective feat yet.
  • Loading branch information
anthogez committed Jul 16, 2021
1 parent d9257d0 commit d21cff7
Show file tree
Hide file tree
Showing 8 changed files with 343 additions and 1 deletion.
3 changes: 3 additions & 0 deletions src/lib/common.ts
@@ -0,0 +1,3 @@
export async function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
129 changes: 129 additions & 0 deletions src/lib/ecosystems/polling.ts
@@ -0,0 +1,129 @@
import * as config from '../config';
import { isCI } from '../is-ci';
import { makeRequest } from '../request/promise';
import { Options } from '../types';

import { assembleQueryString } from '../snyk-test/common';
import { getAuthHeader } from '../api-token';
import { ScanResult } from './types';
import { TestDependenciesResult, TestDepGraphMeta } from '../snyk-test/legacy';
import { sleep } from '../common';

type ResolveAndTestFactsStatus =
| 'CANCELLED'
| 'ERROR'
| 'PENDING'
| 'RUNNING'
| 'OK';

interface PollingTask {
pollInterval: number;
maxAttempts: number;
}

interface ResolveAndTestFactsResponse {
token: string;
pollingTask: PollingTask;
result?: TestDependenciesResult;
meta?: TestDepGraphMeta;
status?: ResolveAndTestFactsStatus;
code?: number;
error?: string;
message?: string;
userMessage?: string;
}

export async function requestPollingToken(
options: Options,
isAsync: boolean,
scanResult: ScanResult,
): Promise<ResolveAndTestFactsResponse> {
const payload = {
method: 'POST',
url: `${config.API}/test-dependencies`,
json: true,
headers: {
'x-is-ci': isCI(),
authorization: getAuthHeader(),
},
body: {
isAsync,
scanResult,
},
qs: assembleQueryString(options),
};
const response = await makeRequest<ResolveAndTestFactsResponse>(payload);
throwIfRequestPollingTokenFailed(response);
return response;
}

function throwIfRequestPollingTokenFailed(res: ResolveAndTestFactsResponse) {
const { token, status, pollingTask } = res;
const { maxAttempts, pollInterval } = pollingTask;
const isMissingPollingTask = !!maxAttempts && !!pollInterval;
if (!token && !status && isMissingPollingTask) {
throw 'Something went wrong, invalid response.';
}
}

export async function pollingWithTokenUntilDone(
token: string,
type: string,
options: Options,
pollInterval: number,
attemptsCount: number,
maxAttempts = Infinity,
): Promise<ResolveAndTestFactsResponse> {
const payload = {
method: 'GET',
url: `${config.API}/test-dependencies/${token}`,
json: true,
headers: {
'x-is-ci': isCI(),
authorization: getAuthHeader(),
},
qs: { ...assembleQueryString(options), type },
};

const response = await makeRequest<ResolveAndTestFactsResponse>(payload);

if (pollingRequestHasFailed(response)) {
throw response;
}

const taskCompleted = response.result && response.meta;
if (taskCompleted) {
return response;
}

attemptsCount++;
checkPollingAttempts(maxAttempts)(attemptsCount);

await sleep(pollInterval);

return await pollingWithTokenUntilDone(
token,
type,
options,
pollInterval,
attemptsCount,
maxAttempts,
);
}

function checkPollingAttempts(maxAttempts: number) {
return (attemptsCount: number) => {
if (attemptsCount > maxAttempts) {
throw new Error('Exceeded Polling maxAttempts');
}
};
}

function pollingRequestHasFailed(
response: ResolveAndTestFactsResponse,
): boolean {
const { token, result, meta, status, error, code, message } = response;
const hasError = !!error && !!code && !!message;
const pollingContextIsMissing = !token && !result && !meta && !status;
return !!pollingContextIsMissing || hasError;
}
49 changes: 49 additions & 0 deletions src/lib/ecosystems/resolve-test-facts.ts
@@ -0,0 +1,49 @@
import { Options } from '../types';
import * as spinner from '../../lib/spinner';
import { Ecosystem, ScanResult, TestResult } from './types';
import { pollingWithTokenUntilDone, requestPollingToken } from './polling';

export async function resolveAndTestFacts(
ecosystem: Ecosystem,
scans: {
[dir: string]: ScanResult[];
},
options: Options,
): Promise<[TestResult[], string[]]> {
const results: any[] = [];
const errors: string[] = [];

for (const [path, scanResults] of Object.entries(scans)) {
await spinner(`Resolving and Testing fileSignatures in ${path}`);
for (const scanResult of scanResults) {
try {
const res = await requestPollingToken(options, true, scanResult);
const { maxAttempts, pollInterval } = res.pollingTask;
const attemptsCount = 0;
const response = await pollingWithTokenUntilDone(
res.token,
ecosystem,
options,
pollInterval,
attemptsCount,
maxAttempts,
);
results.push({
issues: response?.result?.issues,
issuesData: response?.result?.issuesData,
depGraphData: response?.result?.depGraphData,
});
} catch (error) {
const hasStatusCodeError = error.code >= 400 && error.code <= 500;
if (hasStatusCodeError) {
errors.push(error.message);
continue;
}
const failedPath = path ? `in ${path}` : '.';
errors.push(`Could not test dependencies ${failedPath}`);
}
}
}
spinner.clearAll();
return [results, errors];
}
2 changes: 1 addition & 1 deletion src/lib/snyk-test/legacy.ts
Expand Up @@ -236,7 +236,7 @@ interface Issue {
fixInfo: FixInfo;
}

interface TestDependenciesResult {
export interface TestDependenciesResult {
issuesData: {
[issueId: string]: IssueData;
};
Expand Down
36 changes: 36 additions & 0 deletions test/jest/unit/lib/ecosystems/fixtures/depgraph-data.ts
@@ -0,0 +1,36 @@
export const depGraphData = {
schemaVersion: '1.2.0',
pkgManager: { name: 'cpp' },
pkgs: [
{ id: '_root@0.0.0', info: { name: '_root', version: '0.0.0' } },
{
id: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
info: {
name: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip',
version: '0.5.0',
},
},
],
graph: {
rootNodeId: 'root-node',
nodes: [
{
nodeId: 'root-node',
pkgId: '_root@0.0.0',
deps: [
{
nodeId:
'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
},
],
},
{
nodeId:
'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
pkgId: 'fastlz|https://github.com/ariya/fastlz/archive/0.5.0.zip@0.5.0',
deps: [],
},
],
},
};

4 changes: 4 additions & 0 deletions test/jest/unit/lib/ecosystems/fixtures/index.ts
@@ -0,0 +1,4 @@
import { depGraphData } from './depgraph-data';
import { scanResults } from './scan-results';

export { depGraphData, scanResults };
35 changes: 35 additions & 0 deletions test/jest/unit/lib/ecosystems/fixtures/scan-results.ts
@@ -0,0 +1,35 @@
/* eslint-disable @typescript-eslint/camelcase */
export const scanResults = {
path: [
{
name: 'my-unmanaged-c-project',
facts: [
{
type: 'fileSignatures',
data: [
{
path: 'fastlz_example/fastlz.h',
hashes_ffm: [
{
format: 1,
data: 'ucMc383nMM/wkFRM4iOo5Q',
},
{
format: 1,
data: 'k+DxEmslFQWuJsZFXvSoYw',
},
],
},
],
},
],
identity: {
type: 'cpp',
},
target: {
remoteUrl: 'https://github.com/some-org/some-unmanaged-project.git',
branch: 'master',
},
},
],
};
86 changes: 86 additions & 0 deletions test/jest/unit/lib/ecosystems/resolve-test-facts.spec.ts
@@ -0,0 +1,86 @@
import { Options } from '../../../../../src/lib/types';
import * as polling from '../../../../../src/lib/ecosystems/polling';
import { depGraphData, scanResults } from './fixtures/';
import { resolveAndTestFacts } from '../../../../../src/lib/ecosystems/resolve-test-facts';

describe('resolve and test facts', () => {
afterEach(() => jest.restoreAllMocks());

const token =
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjlkNGQyMzg0LWUwMmYtNGZiYS1hNWI1LTRhMjU4MzFlM2JmOCIsInNhcGlVcmwiOiJodHRwOi8vd3d3LmZha2Utc2FwaS11cmwvIiwic3RhcnRUaW1lIjoxNjI2MDk2MTg5NzQ1fQ.fyI15bzeB_HtMvqRIBQdKpKBZgQADwn3sByEk64DzxA';

const pollingTask = {
pollInterval: 30000,
maxAttempts: 25,
};

it('failing to resolve and test file-signatures fact for c/c++ projects', async () => {
const requestPollingTokenSpy = jest.spyOn(polling, 'requestPollingToken');
const pollingWithTokenUntilDoneSpy = jest.spyOn(
polling,
'pollingWithTokenUntilDone',
);

requestPollingTokenSpy.mockResolvedValueOnce({
token,
status: 'ERROR',
pollingTask,
});

pollingWithTokenUntilDoneSpy.mockRejectedValueOnce({
code: 500,
message:
'Internal error (reference: eb9ab16c-1d33-4586-bf99-ef30c144d1f1)',
});

const [testResults, errors] = await resolveAndTestFacts(
'cpp',
scanResults,
{} as Options,
);

expect(testResults).toEqual([]);
expect(errors[0]).toContain(
'Internal error (reference: eb9ab16c-1d33-4586-bf99-ef30c144d1f1)',
);
});

it('successfully resolving and testing file-signatures fact for c/c++ projects', async () => {
const resolveAndTestFactsSpy = jest.spyOn(polling, 'requestPollingToken');
const pollingWithTokenUntilDoneSpy = jest.spyOn(
polling,
'pollingWithTokenUntilDone',
);

resolveAndTestFactsSpy.mockResolvedValueOnce({
token,
status: 'OK',
pollingTask,
});

pollingWithTokenUntilDoneSpy.mockResolvedValueOnce({
token,
pollingTask,
result: {
issuesData: {},
issues: [],
depGraphData,
},
});

const [testResults, errors] = await resolveAndTestFacts(
'cpp',
scanResults,
{} as Options,
);

expect(testResults).toEqual([
{
issuesData: {},
issues: [],
depGraphData,
},
]);
expect(errors).toEqual([]);
});
});

0 comments on commit d21cff7

Please sign in to comment.