Merge pull request #317 from snyk/feat/extract-red-hat-content-sets

[CAP-120] - Extracting content manifests from layers

.gitignore

@@ -9,6 +9,7 @@ yarn.lock
 test/fixtures/ls-output/xxx.txt
 .nyc_output/
 .vscode/
+tsconfig.local.json
 
 # Diagnostic reports (https://nodejs.org/api/report.html) 
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json

lib/extractor/types.ts

@@ -76,3 +76,9 @@ export interface DetectedImageLayers {
   packages;
   layers;
 }
+
+export interface RedHatRepos {
+  [imageLayerIndex: string]: RedHatRepo[];
+}
+
+export type RedHatRepo = string;

lib/inputs/rhel/static.ts

@@ -0,0 +1,36 @@
+import {
+  ExtractAction,
+  ExtractedLayers,
+  RedHatRepo,
+  RedHatRepos,
+} from "../../extractor/types";
+import { streamToJson } from "../../stream-utils";
+
+export const getRedHatReposContentAction: ExtractAction = {
+  actionName: "redhat-content-manifests",
+  filePathMatches: isRedHatContentManifest,
+  callback: streamToJson,
+};
+
+export function getRedHatReposFromExtractedLayers(
+  extractedLayers: ExtractedLayers,
+): RedHatRepos {
+  const reposByLayer = {};
+  for (const filePath in extractedLayers) {
+    if (isRedHatContentManifest(filePath)) {
+      const content = extractedLayers[filePath][
+        "redhat-content-manifests"
+      ] as any;
+      const layerIndex = content?.metadata?.image_layer_index;
+      const repos = content?.content_sets;
+      if (layerIndex && repos) {
+        reposByLayer[layerIndex] = repos as RedHatRepo;
+      }
+    }
+  }
+  return reposByLayer as RedHatRepos;
+}
+
+function isRedHatContentManifest(filePath: string): boolean {
+  return filePath.startsWith("/root/buildinfo/content_manifests/");
+}

test/fixtures/extracted-layers/with-content-manifests.json

@@ -0,0 +1,17 @@
+{
+  "/root/buildinfo/content_manifests/jenkins-agent-maven-35-rhel7-container-v3.11.346-2.json": {
+    "redhat-content-manifests": {
+      "metadata": {
+        "icm_version": 1,
+        "icm_spec": "https://raw.githubusercontent.com/containerbuildsystem/atomic-reactor/master/atomic_reactor/schemas/content_manifest.json",
+        "image_layer_index": 6
+      },
+      "content_sets": [
+        "rhel-7-server-ose-3.11-rpms",
+        "rhel-server-rhscl-7-rpms",
+        "rhel-7-server-rpms"
+      ],
+      "image_contents": []
+    }
+  }
+}

test/lib/extractor/extractor.spec.ts

@@ -0,0 +1,37 @@
+import { extractImageContent } from "../../../lib/extractor";
+import { getRedHatReposContentAction } from "../../../lib/inputs/rhel/static";
+import { ImageType } from "../../../lib/types";
+import { getFixture } from "../../util/index";
+
+describe("extractImageContent", () => {
+  it("extracts red hat repositories information from layers", async () => {
+    const extractedContent = await extractImageContent(
+      ImageType.DockerArchive,
+      getFixture("docker-archives/docker-save/nginx-with-buildinfo.tar"),
+      [getRedHatReposContentAction],
+    );
+
+    const numOfFoundFiles = Object.keys(extractedContent.extractedLayers)
+      .length;
+    expect(numOfFoundFiles).toBe(1);
+
+    expect(
+      extractedContent.extractedLayers[
+        "/root/buildinfo/content_manifests/jenkins-agent-maven-35-rhel7-container-v3.11.346-2.json"
+      ]["redhat-content-manifests"],
+    ).toMatchObject({
+      metadata: {
+        icm_version: 1,
+        icm_spec:
+          "https://raw.githubusercontent.com/containerbuildsystem/atomic-reactor/master/atomic_reactor/schemas/content_manifest.json",
+        image_layer_index: 6,
+      },
+      content_sets: [
+        "rhel-7-server-ose-3.11-rpms",
+        "rhel-server-rhscl-7-rpms",
+        "rhel-7-server-rpms",
+      ],
+      image_contents: [],
+    });
+  });
+});

test/lib/inputs/rhel/static.spec.ts

@@ -0,0 +1,19 @@
+import { getRedHatReposFromExtractedLayers } from "../../../../lib/inputs/rhel/static";
+import { getObjFromFixture } from "../../../util";
+
+describe("getRedHatReposFromExtractedLayers()", () => {
+  it("correctly gets list of repos and image layer index", () => {
+    const extractedLayers = getObjFromFixture(
+      "extracted-layers/with-content-manifests.json",
+    );
+
+    const repos = getRedHatReposFromExtractedLayers(extractedLayers);
+    expect(repos).toMatchObject({
+      "6": [
+        "rhel-7-server-ose-3.11-rpms",
+        "rhel-server-rhscl-7-rpms",
+        "rhel-7-server-rpms",
+      ],
+    });
+  });
+});

test/system/detected-layers/detected-layers.spec.ts

@@ -9,12 +9,7 @@ import {
 } from "../../../lib/extractor/docker-archive/index";
 import { ExtractAction } from "../../../lib/extractor/types";
 import { AutoDetectedUserInstructions, ImageType } from "../../../lib/types";
-import { getFixture } from "../../util";
-
-const getObjFromFixture = (fixturePath) => {
-  const path = getFixture(fixturePath);
-  return JSON.parse(readFileSync(path, { encoding: "utf-8" }));
-};
+import { getFixture, getObjFromFixture } from "../../util";
 
 const expectedNginxPackages = [
   "gnupg1",

test/util/index.ts

@@ -1,3 +1,4 @@
+import { readFileSync } from "fs";
 import { join } from "path";
 
 export function getFixture(fixturePath: string | string[]): string {
@@ -10,3 +11,8 @@ export function getFixture(fixturePath: string | string[]): string {
   }
   return join(__dirname, "..", "fixtures", ...fixturePath);
 }
+
+export function getObjFromFixture(fixturePath) {
+  const path = getFixture(fixturePath);
+  return JSON.parse(readFileSync(path, { encoding: "utf-8" }));
+}