-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* updating trivy plugin versions and commit hash * Editing Trivy plugin in to read for if dev dependencies is included or not as an argument and then altering the trivy scan command according to if that is true or not * Fixed the parser for scanning lock files. Still need to fix the parser for images and fix the processor * updated the artemisdb model to fix column level constraint issue causing errors, but did not create migration for it yet due to some errors I am getting when trying to create the migration due to my environment. Updated the Trivy plugin to fix parsing issues with the changes in the JSON due to a newer version of Trivy being used. Currently finishing the feature for parsing through directories of a project and generating package-lock files for directories that do not have one. Updated the UI for Trivy to say that it is an SCA scanner as well instead of just a container scanner. * Finished package-lock generator. Now generates package-lock files if package.json exists. Code still needs to be refactored/cleaned/split * split up the code for trivy into multiple files for better readability. Also migrated the artemisdb model to reflect database changes. * temporarily disabling image scanning from trivy for a|b testing * updating axios from 1.6.1 to 1.6.5 * Updating cryptography dependency * moving write_npmrc to libs so that it can be a shared component between node dependencies and trivy as they both rely on the same code. Ensures better code re-usability * removing files that have been relocated/deduced into one location * fixing bug with generate_locks and logging errors * formatting files with black auto formatter for python to keep code compliant with coding standards * fixing write_npmrc import for node_dependencies plugin since the file was moved * returning errors and warnings from the package lock generation to inform Users that we detected a missing lock file when we detect their package.json by itself * changing the write_npmrc file to accept a log as a parameter so that the parent param can supply the logging method as it is a shared function * fixing node_dependencies unit tests to match param changes to write_npmrc * fixing unit tests from changes made to write_npmrc * splitting trivy into 2 plugins. Trivy SCA and trivy image scanning * fixing typo * updating lingUI for internationalization * seperating unit tests for trivy backend changes and then fixing black formatting * changing the trivy container API to just trivy so that users can retain regular functionality as before since trivy was the API argument for container images previously. At the UI level it will still display Trivy Container Image * changing trivy_image folder to just trivy to match API changes * updating the log messages to be more specific * running prettier linter on the server.ts to meet coding standards * updating trivy test file name * fixing discrepency between tool key and api name * added unit tests for package-lock generation warnings in trivy tests and made generate_locks functionality return before attempting to perform more logic if there are no package.jsons found in the first place * Updating UI test to look for both trivy plugins * fixing unit tests * does not need the conditional of trivy_sca to be met as this should be only for image scanning * fixing error made when updating UI unit tests * changed json.dump output to not boolean. Removed un-used import. Cleaned up code for including/discluding dev dependency flag for trivy_sca * changing how we add include dev dependency tag and making logging more specific
- Loading branch information
1 parent
79e11bb
commit dbb9389
Showing
25 changed files
with
534 additions
and
304 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| import subprocess | ||
| import os | ||
| from glob import glob | ||
| from engine.plugins.lib import utils | ||
| from engine.plugins.lib.write_npmrc import handle_npmrc_creation | ||
|
|
||
| logger = utils.setup_logging("trivy_sca") | ||
|
|
||
|
|
||
| def install_package_files(include_dev, path, root_path): | ||
| # Create a package-lock.json file if it doesn't already exist | ||
| logger.info( | ||
| f'Generating package-lock.json for {path.replace(root_path, "")} (including dev dependencies: {include_dev})' | ||
| ) | ||
| cmd = [ | ||
| "npm", | ||
| "install", | ||
| "--package-lock-only", # Generate the needed lockfile | ||
| "--legacy-bundling", # Don't dedup dependencies so that we can correctly trace their root in package.json | ||
| "--legacy-peer-deps", # Ignore peer dependencies, which is the NPM 6.x behavior | ||
| "--no-audit", # Don't run an audit | ||
| ] | ||
| if not include_dev: | ||
| cmd.append("--only=prod") | ||
| return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path, check=False) | ||
|
|
||
|
|
||
| def check_package_files(path: str, include_dev: bool) -> tuple: | ||
| """ | ||
| Main Function | ||
| Find all of the package.json files in the repo and build lock files for them if they dont have one already. | ||
| Parses the results and returns them with the errors. | ||
| """ | ||
|
|
||
| errors = [] | ||
| alerts = [] | ||
|
|
||
| # Find and loop through all the package.json files in the path | ||
| files = glob("%s/**/package.json" % path, recursive=True) | ||
|
|
||
| logger.info("Found %d package.json files", len(files)) | ||
|
|
||
| # If there are no package.json files, exit function | ||
| if len(files) == 0: | ||
| return errors, alerts | ||
|
|
||
| # Build a set of all directories containing package files | ||
| paths = set() | ||
| for filename in files: | ||
| paths.add(os.path.dirname(filename)) | ||
|
|
||
| # Write a .npmrc file based on the set of package.json files found | ||
| handle_npmrc_creation(logger, paths) | ||
|
|
||
| # Loop through paths that have a package file and generate a package-lock.json for them (if does not exist) | ||
| for sub_path in paths: | ||
| lockfile = os.path.join(sub_path, "package-lock.json") | ||
| lockfile_missing = not os.path.exists(lockfile) | ||
| if lockfile_missing: | ||
| msg = ( | ||
| f"No package-lock.json file was found in path {sub_path.replace(path, '')}." | ||
| " Please consider creating a package-lock file for this project." | ||
| ) | ||
| logger.warning(msg) | ||
| alerts.append(msg) | ||
| r = install_package_files(include_dev, sub_path, path) | ||
| if r.returncode != 0: | ||
| error = r.stderr.decode("utf-8") | ||
| logger.error(error) | ||
| errors.append(error) | ||
| return errors, alerts | ||
| # Return the results | ||
| return errors, alerts |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| """ | ||
| trivy output parser | ||
| """ | ||
| import json | ||
| from typing import NamedTuple | ||
| from engine.plugins.lib import utils | ||
|
|
||
| logger = utils.setup_logging("trivy") | ||
|
|
||
| DESC_REMEDIATION_SPLIT = "## Recommendation" | ||
|
|
||
|
|
||
| def parse_output(output: list) -> list: | ||
| results = [] | ||
| for item in output: | ||
| source = item["Target"] | ||
| component_type = convert_type(item.get("Type", "N/A")) | ||
| if item.get("Vulnerabilities") is None: | ||
| continue | ||
| cve_set = set() | ||
| for vuln in item["Vulnerabilities"]: | ||
| vuln_id = vuln.get("VulnerabilityID") | ||
| if vuln_id in cve_set: | ||
| continue | ||
| cve_set.add(vuln_id) | ||
| description_result = get_description_and_remediation(vuln.get("Description"), vuln.get("FixedVersion")) | ||
|
|
||
| component = vuln.get("PkgName") | ||
| if vuln.get("InstalledVersion"): | ||
| component = f'{component}-{vuln.get("InstalledVersion")}' | ||
| results.append( | ||
| { | ||
| "component": component, | ||
| "source": source, | ||
| "id": vuln_id, | ||
| "description": description_result.description, | ||
| "severity": vuln.get("Severity", "").lower(), | ||
| "remediation": description_result.remediation, | ||
| "inventory": { | ||
| "component": { | ||
| "name": vuln.get("PkgName"), | ||
| "version": vuln.get("InstalledVersion"), | ||
| "type": component_type, | ||
| }, | ||
| "advisory_ids": sorted( | ||
| list(set(filter(None, [vuln_id, vuln.get("PrimaryURL")] + vuln.get("References", [])))) | ||
| ), | ||
| }, | ||
| } | ||
| ) | ||
| return results | ||
|
|
||
|
|
||
| def convert_type(component_type: str) -> str: | ||
| if component_type == "bundler": | ||
| return "gem" | ||
| return component_type.lower() | ||
|
|
||
|
|
||
| def get_description_and_remediation(description, fixed_version) -> NamedTuple: | ||
| """ | ||
| gets the description and remediation fields after pulling them from the vuln and appending/removing additional info | ||
| :param fixed_version: | ||
| :param description: | ||
| :return: NamedTuple containing the description and remediation | ||
| """ | ||
| result = NamedTuple("DescriptionResult", [("description", str), ("remediation", str)]) | ||
| if not description: | ||
| description = "" | ||
| remediation = "" | ||
| if DESC_REMEDIATION_SPLIT in description: | ||
| des_split = description.split(DESC_REMEDIATION_SPLIT) | ||
| remediation = des_split[1].strip() | ||
| description = des_split[0].strip() | ||
| if fixed_version: | ||
| remediation = f"Fixed Version: {fixed_version}. {remediation}".strip() | ||
| result.description = description | ||
| result.remediation = remediation | ||
| return result | ||
|
|
||
|
|
||
| def convert_output(output_str: str): | ||
| if not output_str: | ||
| return None | ||
| try: | ||
| return json.loads(output_str) | ||
| except json.JSONDecodeError as e: | ||
| logger.error(e) | ||
| return None |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.