From c01a70a0f0623d50f22e2353e8d09b4ca2db4342 Mon Sep 17 00:00:00 2001 From: Zach Hill Date: Mon, 4 Nov 2019 11:42:39 -0800 Subject: [PATCH] merge bill of materials into a single content.json (#14) * Initial work to provide single bill of materials with merges package listings Signed-off-by: Zach Hill * Add packages property to content BoM for future-proofing Signed-off-by: Zach Hill --- dist/index.js | 75 ++++++++++++++++++----- index.js | 75 ++++++++++++++++++----- lib/run_scan | 8 +-- tests/fixtures/content-merge.fixture.json | 75 +++++++++++++++++++++++ tests/index.test.js | 41 ++++++++++++- 5 files changed, 235 insertions(+), 39 deletions(-) create mode 100644 tests/fixtures/content-merge.fixture.json diff --git a/dist/index.js b/dist/index.js index 1b7490e5..b4571561 100644 --- a/dist/index.js +++ b/dist/index.js @@ -62,7 +62,7 @@ async function run() { core.debug((new Date()).toTimeString()); const required_option = {required: true}; - + const billOfMaterialsPath = "./anchore-reports/content.json"; const image_reference = core.getInput('image-reference', required_option); const dockerfile_path = core.getInput('dockerfile-path'); let debug = core.getInput('debug'); @@ -86,8 +86,8 @@ async function run() { // Load the bundle to extract the policy id let custom_policy = fs.readFileSync(bundle_path); - if(custom_policy) { - core.debug('loaded custom bundle ' +custom_policy); + if (custom_policy) { + core.debug('loaded custom bundle ' + custom_policy); custom_policy = JSON.parse(custom_policy); bundle_name = custom_policy.id; if (!bundle_name) { @@ -120,22 +120,22 @@ async function run() { inline_scan_image = "docker.io/anchore/inline-scan:v0.5.1"; } - core.info('Image: ' +image_reference); - core.info('Dockerfile path: ' +dockerfile_path); - core.info('Inline Scan Image: ' +inline_scan_image); - core.info('Debug Output: ' +debug); - core.info('Fail Build: ' +fail_build); - core.info('Include App Packages: ' +include_packages); - core.info('Custom Policy Path: ' +custom_policy_path); + core.info('Image: ' + image_reference); + core.info('Dockerfile path: ' + dockerfile_path); + core.info('Inline Scan Image: ' + inline_scan_image); + core.info('Debug Output: ' + debug); + core.info('Fail Build: ' + fail_build); + core.info('Include App Packages: ' + include_packages); + core.info('Custom Policy Path: ' + custom_policy_path); - core.debug('Policy path for evaluation: ' +policy_bundle_path); - core.debug('Policy name for evaluation: ' +policy_bundle_name); + core.debug('Policy path for evaluation: ' + policy_bundle_path); + core.debug('Policy name for evaluation: ' + policy_bundle_name); let cmd = `${__dirname}/lib/run_scan ${__dirname}/lib ${scan_scriptname} ${inline_scan_image} ${image_reference} ${debug} ${policy_bundle_path} ${policy_bundle_name}`; if (dockerfile_path) { cmd = `${cmd} ${dockerfile_path}` } - core.info('\nAnalyzing image: ' +image_reference); + core.info('\nAnalyzing image: ' + image_reference); execSync(cmd, {stdio: 'inherit'}); let rawdata = fs.readFileSync('./anchore-reports/policy_evaluation.json'); @@ -144,11 +144,21 @@ async function run() { let imageTag = Object.keys(policyEval[0][imageId[0]]); let policyStatus = policyEval[0][imageId[0]][imageTag][0]['status']; - core.setOutput('billofmaterials', './anchore-reports/content-os.json'); + try { + let billOfMaterials = { + "packages": mergeResults(loadContent(findContent("./anchore-reports/"))) + }; + fs.writeFileSync(billOfMaterialsPath, JSON.stringify(billOfMaterials)); + } catch (error) { + core.error("Error constructing bill of materials from anchore output: " + error); + throw error; + } + + core.setOutput('billofmaterials', billOfMaterialsPath); core.setOutput('vulnerabilities', './anchore-reports/vulnerabilities.json'); core.setOutput('policycheck', policyStatus); - if (fail_build === "true" && policyStatus === "fail") { + if (fail_build === "true" && policyStatus === "fail") { core.setFailed("Image failed Anchore policy evaluation"); } @@ -157,7 +167,40 @@ async function run() { } } -module.exports = run; +// Find all 'content-*.json' files in the directory. dirname should include the full path +function findContent(searchDir) { + let contentFiles = []; + let match = /content-.*\.json/; + var dirItems = fs.readdirSync(searchDir); + if (dirItems) { + for (let i = 0; i < dirItems.length; i++) { + if (match.test(dirItems[i])) { + contentFiles.push(`${searchDir}/${dirItems[i]}`); + } + } + } else { + console.log("no dir content found"); + } + + console.log(contentFiles); + return contentFiles; +} + +// Load the json content of each file in a list and return them as a list +function loadContent(files) { + let contents = []; + if (files) { + files.forEach(item => contents.push(JSON.parse(fs.readFileSync(item)))); + } + return contents +} + +// Merge the multiple content output types into a single array +function mergeResults(contentArray) { + return contentArray.reduce((merged, n) => merged.concat(n.content), []); +} + +module.exports = {run, mergeResults, findContent, loadContent}; if (require.main === require.cache[eval('__filename')]) { run(); diff --git a/index.js b/index.js index 06683730..475c5a10 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ async function run() { core.debug((new Date()).toTimeString()); const required_option = {required: true}; - + const billOfMaterialsPath = "./anchore-reports/content.json"; const image_reference = core.getInput('image-reference', required_option); const dockerfile_path = core.getInput('dockerfile-path'); let debug = core.getInput('debug'); @@ -31,8 +31,8 @@ async function run() { // Load the bundle to extract the policy id let custom_policy = fs.readFileSync(bundle_path); - if(custom_policy) { - core.debug('loaded custom bundle ' +custom_policy); + if (custom_policy) { + core.debug('loaded custom bundle ' + custom_policy); custom_policy = JSON.parse(custom_policy); bundle_name = custom_policy.id; if (!bundle_name) { @@ -65,22 +65,22 @@ async function run() { inline_scan_image = "docker.io/anchore/inline-scan:v0.5.1"; } - core.info('Image: ' +image_reference); - core.info('Dockerfile path: ' +dockerfile_path); - core.info('Inline Scan Image: ' +inline_scan_image); - core.info('Debug Output: ' +debug); - core.info('Fail Build: ' +fail_build); - core.info('Include App Packages: ' +include_packages); - core.info('Custom Policy Path: ' +custom_policy_path); + core.info('Image: ' + image_reference); + core.info('Dockerfile path: ' + dockerfile_path); + core.info('Inline Scan Image: ' + inline_scan_image); + core.info('Debug Output: ' + debug); + core.info('Fail Build: ' + fail_build); + core.info('Include App Packages: ' + include_packages); + core.info('Custom Policy Path: ' + custom_policy_path); - core.debug('Policy path for evaluation: ' +policy_bundle_path); - core.debug('Policy name for evaluation: ' +policy_bundle_name); + core.debug('Policy path for evaluation: ' + policy_bundle_path); + core.debug('Policy name for evaluation: ' + policy_bundle_name); let cmd = `${__dirname}/lib/run_scan ${__dirname}/lib ${scan_scriptname} ${inline_scan_image} ${image_reference} ${debug} ${policy_bundle_path} ${policy_bundle_name}`; if (dockerfile_path) { cmd = `${cmd} ${dockerfile_path}` } - core.info('\nAnalyzing image: ' +image_reference); + core.info('\nAnalyzing image: ' + image_reference); execSync(cmd, {stdio: 'inherit'}); let rawdata = fs.readFileSync('./anchore-reports/policy_evaluation.json'); @@ -89,11 +89,21 @@ async function run() { let imageTag = Object.keys(policyEval[0][imageId[0]]); let policyStatus = policyEval[0][imageId[0]][imageTag][0]['status']; - core.setOutput('billofmaterials', './anchore-reports/content-os.json'); + try { + let billOfMaterials = { + "packages": mergeResults(loadContent(findContent("./anchore-reports/"))) + }; + fs.writeFileSync(billOfMaterialsPath, JSON.stringify(billOfMaterials)); + } catch (error) { + core.error("Error constructing bill of materials from anchore output: " + error); + throw error; + } + + core.setOutput('billofmaterials', billOfMaterialsPath); core.setOutput('vulnerabilities', './anchore-reports/vulnerabilities.json'); core.setOutput('policycheck', policyStatus); - if (fail_build === "true" && policyStatus === "fail") { + if (fail_build === "true" && policyStatus === "fail") { core.setFailed("Image failed Anchore policy evaluation"); } @@ -102,7 +112,40 @@ async function run() { } } -module.exports = run; +// Find all 'content-*.json' files in the directory. dirname should include the full path +function findContent(searchDir) { + let contentFiles = []; + let match = /content-.*\.json/; + var dirItems = fs.readdirSync(searchDir); + if (dirItems) { + for (let i = 0; i < dirItems.length; i++) { + if (match.test(dirItems[i])) { + contentFiles.push(`${searchDir}/${dirItems[i]}`); + } + } + } else { + console.log("no dir content found"); + } + + console.log(contentFiles); + return contentFiles; +} + +// Load the json content of each file in a list and return them as a list +function loadContent(files) { + let contents = []; + if (files) { + files.forEach(item => contents.push(JSON.parse(fs.readFileSync(item)))); + } + return contents +} + +// Merge the multiple content output types into a single array +function mergeResults(contentArray) { + return contentArray.reduce((merged, n) => merged.concat(n.content), []); +} + +module.exports = {run, mergeResults, findContent, loadContent}; if (require.main === module) { run(); diff --git a/lib/run_scan b/lib/run_scan index 01f9bbf1..1fac1db2 100755 --- a/lib/run_scan +++ b/lib/run_scan @@ -86,10 +86,10 @@ docker exec -t local-anchore-engine anchore-cli --json evaluate check --detail $ set -eo pipefail docker exec -t local-anchore-engine anchore-cli --json image vuln ${IMAGE_DIGEST_SHA} all > ./anchore-reports/vulnerabilities.json docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} os > ./anchore-reports/content-os.json -# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} java > ./anchore-reports/content-java.json -# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} gem > ./anchore-reports/content-gem.json -# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} python > ./anchore-reports/content-python.json -# docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} npm > ./anchore-reports/content-npm.json +docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} java > ./anchore-reports/content-java.json +docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} gem > ./anchore-reports/content-gem.json +docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} python > ./anchore-reports/content-python.json +docker exec -t local-anchore-engine anchore-cli --json image content ${IMAGE_DIGEST_SHA} npm > ./anchore-reports/content-npm.json echo "" if [ "${debug}" = "true" ]; then diff --git a/tests/fixtures/content-merge.fixture.json b/tests/fixtures/content-merge.fixture.json new file mode 100644 index 00000000..12947fdb --- /dev/null +++ b/tests/fixtures/content-merge.fixture.json @@ -0,0 +1,75 @@ +{ + "content-gem.json": { + "content": [ + { + "license": "ruby", + "location": "/usr/lib/ruby/gems/2.3.0/specifications/default/bigdecimal-1.2.8.gemspec", + "origin": "Kenta Murata,Zachary Scott,Shigeo Kobayashi", + "package": "bigdecimal", + "type": "GEM", + "version": "1.2.8" + } + ], + "content_type": "gem", + "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea" + }, + "content-java.json": { + "content": [ + { + "implementation-version": "N/A", + "location": "/usr/share/java/java-atk-wrapper.jar", + "maven-version": "N/A", + "origin": "N/A", + "package": "java-atk-wrapper", + "specification-version": "N/A", + "type": "JAVA-JAR" + } + ], + "content_type": "java", + "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea" + }, + "content-npm.json": { + "content": [ + { + "license": "BSD-2-Clause", + "location": "/opt/yarn-v1.19.1/package.json", + "origin": "Unknown", + "package": "yarn", + "type": "NPM", + "version": "1.19.1" + } + ], + "content_type": "npm", + "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea" + }, + "content-python.json": { + "content": [ + { + "license": "Python Software Foundation License", + "location": "/usr/lib/python2.7", + "origin": "Steven Bethard ", + "package": "argparse", + "type": "PYTHON", + "version": "1.2.1" + } + ], + "content_type": "python", + "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea" + }, + "content-os.json": { + "content": [ + { + "license": "GPLv2+", + "origin": "APT Development Team (maintainer)", + "package": "apt", + "size": "3539000", + "type": "dpkg", + "version": "1.4.9" + } + ], + "content_type": "os", + "imageDigest": "sha256:0a97ccb2868e3c54167317fe7a2fc58e5123290d6c5b653a725091cbf18ca1ea" + } + + +} \ No newline at end of file diff --git a/tests/index.test.js b/tests/index.test.js index 11d80cdc..d8250d2b 100644 --- a/tests/index.test.js +++ b/tests/index.test.js @@ -6,9 +6,11 @@ const _ = require('lodash'); const core = require('@actions/core'); const child_process = require('child_process'); const fs = require('fs'); +const path = require('path'); -const run = require('..'); +const main = require('..'); const policyEvaluationFixture = require('./fixtures/policy_evaluation.fixture'); +const contentMergeFixture = require('./fixtures/content-merge.fixture'); describe('anchore-scan-action', () => { beforeEach(() => { @@ -30,7 +32,7 @@ describe('anchore-scan-action', () => { }); core.setFailed = jest.fn(); - await run(); + await main.run(); expect(core.setFailed).not.toHaveBeenCalled(); }); @@ -44,8 +46,41 @@ describe('anchore-scan-action', () => { }); core.setFailed = jest.fn(); - await run(); + await main.run(); expect(core.setFailed).toHaveBeenCalled(); }); + + it('tests merge of outputs into single bill of materials with os-only packages', async () => { + let merged = main.mergeResults([contentMergeFixture["content-os.json"]]); + //console.log("os-only output: " +JSON.stringify(merged)); + expect(merged.length).toBeGreaterThan(0); + + }); + + it('tests merge of outputs into single bill of materials with all packages', async () => { + let merged = main.mergeResults([contentMergeFixture["content-os.json"], contentMergeFixture["content-npm.json"], contentMergeFixture["content-gem.json"], contentMergeFixture["content-java.json"], contentMergeFixture["content-python.json"]]); + //console.log("merged output: " +JSON.stringify(merged)); + expect(merged.length).toBeGreaterThan(0); + }); + + it('tests finding content files in dir', async () => { + let testPath = path.join(__dirname, "fixtures"); + fs.readdirSync = jest.fn(() => { + return Object.keys(contentMergeFixture); + }); + + let contentFiles = main.findContent(testPath); + expect(contentFiles.length).toEqual(5); + }); + + it('tests loading content in list', async () => { + fs.readFileSync = jest.fn((i) => { + return JSON.stringify(contentMergeFixture[i]); + }); + + let contentFiles = main.loadContent(Object.keys(contentMergeFixture)); + expect(contentFiles.length).toEqual(5); + console.log(contentFiles); + }); });