merge bill of materials into a single content.json (#14)

* Initial work to provide single bill of materials with merges package listings Signed-off-by: Zach Hill <[email protected]> * Add packages property to content BoM for future-proofing Signed-off-by: Zach Hill <[email protected]>
anchore · Nov 4, 2019 · c01a70a · c01a70a
1 parent 3ff3012
commit c01a70a
Show file tree

Hide file tree

Showing 5 changed files with 235 additions and 39 deletions.
diff --git a/dist/index.js b/dist/index.js
@@ -62,7 +62,7 @@ async function run() {
         core.debug((new Date()).toTimeString());
 
         const required_option = {required: true};
-
+        const billOfMaterialsPath = "./anchore-reports/content.json";
         const image_reference = core.getInput('image-reference', required_option);
         const dockerfile_path = core.getInput('dockerfile-path');
         let debug = core.getInput('debug');
@@ -86,8 +86,8 @@ async function run() {
             // Load the bundle to extract the policy id
             let custom_policy = fs.readFileSync(bundle_path);
 
-            if(custom_policy) {
-                core.debug('loaded custom bundle ' +custom_policy);
+            if (custom_policy) {
+                core.debug('loaded custom bundle ' + custom_policy);
                 custom_policy = JSON.parse(custom_policy);
                 bundle_name = custom_policy.id;
                 if (!bundle_name) {
@@ -120,22 +120,22 @@ async function run() {
             inline_scan_image = "docker.io/anchore/inline-scan:v0.5.1";
         }
 
-        core.info('Image: ' +image_reference);
-        core.info('Dockerfile path: ' +dockerfile_path);
-        core.info('Inline Scan Image: ' +inline_scan_image);
-        core.info('Debug Output: ' +debug);
-        core.info('Fail Build: ' +fail_build);
-        core.info('Include App Packages: ' +include_packages);
-        core.info('Custom Policy Path: ' +custom_policy_path);
+        core.info('Image: ' + image_reference);
+        core.info('Dockerfile path: ' + dockerfile_path);
+        core.info('Inline Scan Image: ' + inline_scan_image);
+        core.info('Debug Output: ' + debug);
+        core.info('Fail Build: ' + fail_build);
+        core.info('Include App Packages: ' + include_packages);
+        core.info('Custom Policy Path: ' + custom_policy_path);
 
-        core.debug('Policy path for evaluation: ' +policy_bundle_path);
-        core.debug('Policy name for evaluation: ' +policy_bundle_name);
+        core.debug('Policy path for evaluation: ' + policy_bundle_path);
+        core.debug('Policy name for evaluation: ' + policy_bundle_name);
 
         let cmd = `${__dirname}/lib/run_scan ${__dirname}/lib ${scan_scriptname} ${inline_scan_image} ${image_reference} ${debug} ${policy_bundle_path} ${policy_bundle_name}`;
         if (dockerfile_path) {
             cmd = `${cmd} ${dockerfile_path}`
         }
-        core.info('\nAnalyzing image: ' +image_reference);
+        core.info('\nAnalyzing image: ' + image_reference);
         execSync(cmd, {stdio: 'inherit'});
 
         let rawdata = fs.readFileSync('./anchore-reports/policy_evaluation.json');
@@ -144,11 +144,21 @@ async function run() {
         let imageTag = Object.keys(policyEval[0][imageId[0]]);
         let policyStatus = policyEval[0][imageId[0]][imageTag][0]['status'];
 
-        core.setOutput('billofmaterials', './anchore-reports/content-os.json');
+        try {
+            let billOfMaterials = {
+                "packages": mergeResults(loadContent(findContent("./anchore-reports/")))
+            };
+            fs.writeFileSync(billOfMaterialsPath, JSON.stringify(billOfMaterials));
+        } catch (error) {
+            core.error("Error constructing bill of materials from anchore output: " + error);
+            throw error;
+        }
+
+        core.setOutput('billofmaterials', billOfMaterialsPath);
         core.setOutput('vulnerabilities', './anchore-reports/vulnerabilities.json');
         core.setOutput('policycheck', policyStatus);
 
-        if (fail_build === "true"  && policyStatus === "fail") {
+        if (fail_build === "true" && policyStatus === "fail") {
             core.setFailed("Image failed Anchore policy evaluation");
         }
 
@@ -157,7 +167,40 @@ async function run() {
     }
 }
 
-module.exports = run;
+// Find all 'content-*.json' files in the directory. dirname should include the full path
+function findContent(searchDir) {
+    let contentFiles = [];
+    let match = /content-.*\.json/;
+    var dirItems = fs.readdirSync(searchDir);
+    if (dirItems) {
+        for (let i = 0; i < dirItems.length; i++) {
+            if (match.test(dirItems[i])) {
+                contentFiles.push(`${searchDir}/${dirItems[i]}`);
+            }
+        }
+    } else {
+        console.log("no dir content found");
+    }
+
+    console.log(contentFiles);
+    return contentFiles;
+}
+
+// Load the json content of each file in a list and return them as a list
+function loadContent(files) {
+    let contents = [];
+    if (files) {
+        files.forEach(item => contents.push(JSON.parse(fs.readFileSync(item))));
+    }
+    return contents
+}
+
+// Merge the multiple content output types into a single array
+function mergeResults(contentArray) {
+    return contentArray.reduce((merged, n) => merged.concat(n.content), []);
+}
+
+module.exports = {run, mergeResults, findContent, loadContent};
 
 if (require.main === require.cache[eval('__filename')]) {
     run();

diff --git a/index.js b/index.js
@@ -7,7 +7,7 @@ async function run() {
         core.debug((new Date()).toTimeString());
 
         const required_option = {required: true};
-
+        const billOfMaterialsPath = "./anchore-reports/content.json";
         const image_reference = core.getInput('image-reference', required_option);
         const dockerfile_path = core.getInput('dockerfile-path');
         let debug = core.getInput('debug');
@@ -31,8 +31,8 @@ async function run() {
             // Load the bundle to extract the policy id
             let custom_policy = fs.readFileSync(bundle_path);
 
-            if(custom_policy) {
-                core.debug('loaded custom bundle ' +custom_policy);
+            if (custom_policy) {
+                core.debug('loaded custom bundle ' + custom_policy);
                 custom_policy = JSON.parse(custom_policy);
                 bundle_name = custom_policy.id;
                 if (!bundle_name) {
@@ -65,22 +65,22 @@ async function run() {
             inline_scan_image = "docker.io/anchore/inline-scan:v0.5.1";
         }
 
-        core.info('Image: ' +image_reference);
-        core.info('Dockerfile path: ' +dockerfile_path);
-        core.info('Inline Scan Image: ' +inline_scan_image);
-        core.info('Debug Output: ' +debug);
-        core.info('Fail Build: ' +fail_build);
-        core.info('Include App Packages: ' +include_packages);
-        core.info('Custom Policy Path: ' +custom_policy_path);
+        core.info('Image: ' + image_reference);
+        core.info('Dockerfile path: ' + dockerfile_path);
+        core.info('Inline Scan Image: ' + inline_scan_image);
+        core.info('Debug Output: ' + debug);
+        core.info('Fail Build: ' + fail_build);
+        core.info('Include App Packages: ' + include_packages);
+        core.info('Custom Policy Path: ' + custom_policy_path);
 
-        core.debug('Policy path for evaluation: ' +policy_bundle_path);
-        core.debug('Policy name for evaluation: ' +policy_bundle_name);
+        core.debug('Policy path for evaluation: ' + policy_bundle_path);
+        core.debug('Policy name for evaluation: ' + policy_bundle_name);
 
         let cmd = `${__dirname}/lib/run_scan ${__dirname}/lib ${scan_scriptname} ${inline_scan_image} ${image_reference} ${debug} ${policy_bundle_path} ${policy_bundle_name}`;
         if (dockerfile_path) {
             cmd = `${cmd} ${dockerfile_path}`
         }
-        core.info('\nAnalyzing image: ' +image_reference);
+        core.info('\nAnalyzing image: ' + image_reference);
         execSync(cmd, {stdio: 'inherit'});
 
         let rawdata = fs.readFileSync('./anchore-reports/policy_evaluation.json');
@@ -89,11 +89,21 @@ async function run() {
         let imageTag = Object.keys(policyEval[0][imageId[0]]);
         let policyStatus = policyEval[0][imageId[0]][imageTag][0]['status'];
 
-        core.setOutput('billofmaterials', './anchore-reports/content-os.json');
+        try {
+            let billOfMaterials = {
+                "packages": mergeResults(loadContent(findContent("./anchore-reports/")))
+            };
+            fs.writeFileSync(billOfMaterialsPath, JSON.stringify(billOfMaterials));
+        } catch (error) {
+            core.error("Error constructing bill of materials from anchore output: " + error);
+            throw error;
+        }
+
+        core.setOutput('billofmaterials', billOfMaterialsPath);
         core.setOutput('vulnerabilities', './anchore-reports/vulnerabilities.json');
         core.setOutput('policycheck', policyStatus);
 
-        if (fail_build === "true"  && policyStatus === "fail") {
+        if (fail_build === "true" && policyStatus === "fail") {
             core.setFailed("Image failed Anchore policy evaluation");
         }
 
@@ -102,7 +112,40 @@ async function run() {
     }
 }
 
-module.exports = run;
+// Find all 'content-*.json' files in the directory. dirname should include the full path
+function findContent(searchDir) {
+    let contentFiles = [];
+    let match = /content-.*\.json/;
+    var dirItems = fs.readdirSync(searchDir);
+    if (dirItems) {
+        for (let i = 0; i < dirItems.length; i++) {
+            if (match.test(dirItems[i])) {
+                contentFiles.push(`${searchDir}/${dirItems[i]}`);
+            }
+        }
+    } else {
+        console.log("no dir content found");
+    }
+
+    console.log(contentFiles);
+    return contentFiles;
+}
+
+// Load the json content of each file in a list and return them as a list
+function loadContent(files) {
+    let contents = [];
+    if (files) {
+        files.forEach(item => contents.push(JSON.parse(fs.readFileSync(item))));
+    }
+    return contents
+}
+
+// Merge the multiple content output types into a single array
+function mergeResults(contentArray) {
+    return contentArray.reduce((merged, n) => merged.concat(n.content), []);
+}
+
+module.exports = {run, mergeResults, findContent, loadContent};
 
 if (require.main === module) {
     run();

diff --git a/lib/run_scan b/lib/run_scan
@@ -86,10 +86,10 @@ docker exec -t local-anchore-engine anchore-cli --json evaluate check --detail $
 set -eo pipefail
 docker exec -t local-anchore-engine anchore-cli --json image vuln ${IMAGE_DIGEST_SHA} all > ./anchore-reports/vulnerabilities.json
 docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} os > ./anchore-reports/content-os.json
-# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} java > ./anchore-reports/content-java.json
-# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} gem > ./anchore-reports/content-gem.json
-# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} python > ./anchore-reports/content-python.json
-# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} npm > ./anchore-reports/content-npm.json
+docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} java > ./anchore-reports/content-java.json
+docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} gem > ./anchore-reports/content-gem.json
+docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} python > ./anchore-reports/content-python.json
+docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} npm > ./anchore-reports/content-npm.json
 echo ""
 
 if [ "${debug}" = "true" ]; then

diff --git a/tests/fixtures/content-merge.fixture.json b/tests/fixtures/content-merge.fixture.json
@@ -0,0 +1,75 @@
+{
+  "content-gem.json": {
+    "content": [
+      {
+        "license": "ruby",
+        "location": "/usr/lib/ruby/gems/2.3.0/specifications/default/bigdecimal-1.2.8.gemspec",
+        "origin": "Kenta Murata,Zachary Scott,Shigeo Kobayashi",
+        "package": "bigdecimal",
+        "type": "GEM",
+        "version": "1.2.8"
+      }
+    ],
+    "content_type": "gem",
+    "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea"
+  },
+  "content-java.json": {
+    "content": [
+      {
+        "implementation-version": "N/A",
+        "location": "/usr/share/java/java-atk-wrapper.jar",
+        "maven-version": "N/A",
+        "origin": "N/A",
+        "package": "java-atk-wrapper",
+        "specification-version": "N/A",
+        "type": "JAVA-JAR"
+      }
+    ],
+    "content_type": "java",
+    "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea"
+  },
+  "content-npm.json": {
+    "content": [
+      {
+        "license": "BSD-2-Clause",
+        "location": "/opt/yarn-v1.19.1/package.json",
+        "origin": "Unknown",
+        "package": "yarn",
+        "type": "NPM",
+        "version": "1.19.1"
+      }
+    ],
+    "content_type": "npm",
+    "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea"
+  },
+  "content-python.json": {
+    "content": [
+      {
+        "license": "Python Software Foundation License",
+        "location": "/usr/lib/python2.7",
+        "origin": "Steven Bethard <[email protected]>",
+        "package": "argparse",
+        "type": "PYTHON",
+        "version": "1.2.1"
+      }
+    ],
+    "content_type": "python",
+    "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea"
+  },
+  "content-os.json": {
+    "content": [
+      {
+        "license": "GPLv2+",
+        "origin": "APT Development Team <[email protected]> (maintainer)",
+        "package": "apt",
+        "size": "3539000",
+        "type": "dpkg",
+        "version": "1.4.9"
+      }
+    ],
+    "content_type": "os",
+    "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea"
+  }
+
+
+}
diff --git a/tests/index.test.js b/tests/index.test.js
@@ -6,9 +6,11 @@ const _ = require('lodash');
 const core = require('@actions/core');
 const child_process = require('child_process');
 const fs = require('fs');
+const path = require('path');
 
-const run = require('..');
+const main = require('..');
 const policyEvaluationFixture = require('./fixtures/policy_evaluation.fixture');
+const contentMergeFixture = require('./fixtures/content-merge.fixture');
 
 describe('anchore-scan-action', () => {
     beforeEach(() => {
@@ -30,7 +32,7 @@ describe('anchore-scan-action', () => {
         });
         core.setFailed = jest.fn();
 
-        await run();
+        await main.run();
 
         expect(core.setFailed).not.toHaveBeenCalled();
     });
@@ -44,8 +46,41 @@ describe('anchore-scan-action', () => {
         });
         core.setFailed = jest.fn();
 
-        await run();
+        await main.run();
 
         expect(core.setFailed).toHaveBeenCalled();
     });
+
+    it('tests merge of outputs into single bill of materials with os-only packages', async () => {
+        let merged = main.mergeResults([contentMergeFixture["content-os.json"]]);
+        //console.log("os-only output: " +JSON.stringify(merged));
+        expect(merged.length).toBeGreaterThan(0);
+
+    });
+
+    it('tests merge of outputs into single bill of materials with all packages', async () => {
+        let merged = main.mergeResults([contentMergeFixture["content-os.json"], contentMergeFixture["content-npm.json"], contentMergeFixture["content-gem.json"], contentMergeFixture["content-java.json"], contentMergeFixture["content-python.json"]]);
+        //console.log("merged output: " +JSON.stringify(merged));
+        expect(merged.length).toBeGreaterThan(0);
+    });
+
+    it('tests finding content files in dir', async () => {
+        let testPath = path.join(__dirname, "fixtures");
+        fs.readdirSync = jest.fn(() => {
+            return Object.keys(contentMergeFixture);
+        });
+
+        let contentFiles = main.findContent(testPath);
+        expect(contentFiles.length).toEqual(5);
+    });
+
+    it('tests loading content in list', async () => {
+        fs.readFileSync = jest.fn((i) => {
+            return JSON.stringify(contentMergeFixture[i]);
+        });
+
+        let contentFiles = main.loadContent(Object.keys(contentMergeFixture));
+        expect(contentFiles.length).toEqual(5);
+        console.log(contentFiles);
+    });
 });