diff --git a/.github/workflows/github-actions.yml b/.github/workflows/ci.yml similarity index 100% rename from .github/workflows/github-actions.yml rename to .github/workflows/ci.yml diff --git a/.github/workflows/vdr-creation.yml b/.github/workflows/vdr-creation.yml new file mode 100644 index 0000000..f592052 --- /dev/null +++ b/.github/workflows/vdr-creation.yml @@ -0,0 +1,43 @@ +name: VDR Creation + +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: # prs don't get secrets, but the API works (albeit 10x slower) without the api key + branches: [ main ] + + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + + - name: Set up Python 3.10 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Download ojvg + env: + NIST_NVD_TOKEN: ${{ secrets.NIST_NVD_TOKEN }} + run: | + mkdir -p data + python3 ojvg_download.py + python3 ojvg_convert.py + - name: Upload data directory (for debugging/introspection) + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: data directory + path: data + - name: Upload final vdr + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: final vdr + path: data/vdr.json diff --git a/cvereporter/fetch_vulnerabilities.py b/cvereporter/fetch_vulnerabilities.py index f832469..a7b5b34 100644 --- a/cvereporter/fetch_vulnerabilities.py +++ b/cvereporter/fetch_vulnerabilities.py @@ -102,7 +102,9 @@ def parse_to_dict(resp_text: str, date: str) -> list[dict]: # find the table with the CVEs table = soup.find("table", attrs={"class": "risk-matrix"}) - + if table is None: + print("unable to find risk matrix for "+date) + return None # find all the rows in the table rows = table.find_all("tr") dicts = [] @@ -180,8 +182,8 @@ def dict_to_vulns(dicts: list[dict]) -> list[Vulnerability]: """ We assume the text for the affected versions is in a block like: -"The following vulnerabilities in OpenJDK source code were fixed in this release. -The affected versions are 12, 11.0.2, 8u202, 7u211, and earlier. +"The following vulnerabilities in OpenJDK source code were fixed in this release. +The affected versions are 12, 11.0.2, 8u202, 7u211, and earlier. We recommend that you upgrade as soon as possible." """ diff --git a/cvereporter/nist_enhance.py b/cvereporter/nist_enhance.py index 7a8a9fc..ca84639 100644 --- a/cvereporter/nist_enhance.py +++ b/cvereporter/nist_enhance.py @@ -6,6 +6,8 @@ ) import requests import json +import time +import os """ this file has the utilities for downloading data about cves from NIST and updating Vulnerability objects with the data @@ -14,7 +16,15 @@ def fetch_nist(url: str, id: str) -> dict: data = None - nist_resp = requests.get(url) + nist_resp = None + if "NIST_NVD_TOKEN" in os.environ and os.environ["NIST_NVD_TOKEN"]: # check not empty + print("making call to NIST using api key! "+url, flush=True) + time.sleep(1) # stay well within 50 requests/30 seconds + nist_resp = requests.get(url, headers= {"apiKey": os.environ["NIST_NVD_TOKEN"]}) + else: + print("making call to NIST without using api key! "+url, flush=True) + time.sleep(10) # stay well within 5 requests/30 seconds + nist_resp = requests.get(url) if nist_resp.status_code != 200: print( "error fetching {}; status code: {}; text: {}".format(