BaseTools/BuildReport: Improve compile_commands generation #28
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs CodeQL against the repository. | |
| # | |
| # Results are uploaded to GitHub Code Scanning. | |
| # | |
| # Due to a known issue with the CodeQL extractor when building the edk2 | |
| # codebase on Linux systems, only Windows agents are used for build with | |
| # the VS toolchain. | |
| # | |
| # Copyright (c) Microsoft Corporation. | |
| # SPDX-License-Identifier: BSD-2-Clause-Patent | |
| name: "CodeQL" | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - '!**.c' | |
| - '!**.h' | |
| jobs: | |
| analyze: | |
| name: Analyze | |
| runs-on: windows-2019 | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - Package: "ArmPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "CryptoPkg" | |
| ArchList: "IA32" | |
| - Package: "CryptoPkg" | |
| ArchList: "X64" | |
| - Package: "DynamicTablesPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "FatPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "FmpDevicePkg" | |
| ArchList: "IA32,X64" | |
| - Package: "IntelFsp2Pkg" | |
| ArchList: "IA32,X64" | |
| - Package: "IntelFsp2WrapperPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "MdeModulePkg" | |
| ArchList: "IA32" | |
| - Package: "MdeModulePkg" | |
| ArchList: "X64" | |
| - Package: "MdePkg" | |
| ArchList: "IA32,X64" | |
| - Package: "PcAtChipsetPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "PrmPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "SecurityPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "ShellPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "SourceLevelDebugPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "StandaloneMmPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "UefiCpuPkg" | |
| ArchList: "IA32,X64" | |
| - Package: "UnitTestFrameworkPkg" | |
| ArchList: "IA32,X64" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: 'pip-requirements.txt' | |
| - name: Use Git Long Paths on Windows | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: | | |
| git config --system core.longpaths true | |
| - name: Install/Upgrade pip Modules | |
| run: pip install -r pip-requirements.txt --upgrade requests sarif-tools | |
| - name: Determine CI Settings File Supported Operations | |
| id: get_ci_file_operations | |
| shell: python | |
| run: | | |
| import importlib | |
| import os | |
| import sys | |
| from pathlib import Path | |
| from edk2toolext.invocables.edk2_ci_setup import CiSetupSettingsManager | |
| from edk2toolext.invocables.edk2_setup import SetupSettingsManager | |
| # Find the repo CI Settings file | |
| ci_settings_file = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('.pytool/CISettings.py')) | |
| # Note: At this point, submodules have not been pulled, only one CI Settings file should exist | |
| if len(ci_settings_file) != 1 or not ci_settings_file[0].is_file(): | |
| print("::error title=Workspace Error!::Failed to find CI Settings file!") | |
| sys.exit(1) | |
| ci_settings_file = ci_settings_file[0] | |
| # Try Finding the Settings class in the file | |
| module_name = 'ci_settings' | |
| spec = importlib.util.spec_from_file_location(module_name, ci_settings_file) | |
| module = importlib.util.module_from_spec(spec) | |
| spec.loader.exec_module(module) | |
| try: | |
| settings = getattr(module, 'Settings') | |
| except AttributeError: | |
| print("::error title=Workspace Error!::Failed to find Settings class in CI Settings file!") | |
| sys.exit(1) | |
| # Determine Which Operations Are Supported by the Settings Class | |
| ci_setup_supported = issubclass(settings, CiSetupSettingsManager) | |
| setup_supported = issubclass(settings, SetupSettingsManager) | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| print(f'ci_setup_supported={str(ci_setup_supported).lower()}', file=fh) | |
| print(f'setup_supported={str(setup_supported).lower()}', file=fh) | |
| - name: Convert Arch to Log Format | |
| id: convert_arch_hyphen | |
| env: | |
| ARCH_LIST: ${{ matrix.ArchList }} | |
| shell: python | |
| run: | | |
| import os | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| print(f'arch_list={os.environ["ARCH_LIST"].replace(",", "-")}', file=fh) | |
| - name: Setup | |
| if: steps.get_ci_file_operations.outputs.setup_supported == 'true' | |
| run: stuart_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 | |
| - name: Upload Setup Log As An Artifact | |
| uses: actions/upload-artifact@v4 | |
| if: (success() || failure()) && steps.get_ci_file_operations.outputs.setup_supported == 'true' | |
| with: | |
| name: ${{ matrix.Package }}-${{ steps.convert_arch_hyphen.outputs.arch_list }}-Setup-Log | |
| path: | | |
| **/SETUPLOG.txt | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: CI Setup | |
| if: steps.get_ci_file_operations.outputs.ci_setup_supported == 'true' | |
| run: stuart_ci_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 | |
| - name: Upload CI Setup Log As An Artifact | |
| uses: actions/upload-artifact@v4 | |
| if: (success() || failure()) && steps.get_ci_file_operations.outputs.ci_setup_supported == 'true' | |
| with: | |
| name: ${{ matrix.Package }}-${{ steps.convert_arch_hyphen.outputs.arch_list }}-CI-Setup-Log | |
| path: | | |
| **/CISETUP.txt | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Update | |
| run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 | |
| - name: Upload Update Log As An Artifact | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() | |
| with: | |
| name: ${{ matrix.Package }}-${{ steps.convert_arch_hyphen.outputs.arch_list }}-Update-Log | |
| path: | | |
| **/UPDATE_LOG.txt | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Build Tools From Source | |
| run: python BaseTools/Edk2ToolsBuild.py -t VS2019 | |
| - name: Find CodeQL Plugin Directory | |
| id: find_dir | |
| shell: python | |
| run: | | |
| import os | |
| import sys | |
| from pathlib import Path | |
| # Find the plugin directory that contains the CodeQL plugin | |
| plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('BaseTools/Plugin/CodeQL')) | |
| # This should only be found once | |
| if len(plugin_dir) == 1: | |
| plugin_dir = str(plugin_dir[0]) | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| print(f'codeql_plugin_dir={plugin_dir}', file=fh) | |
| else: | |
| print("::error title=Workspace Error!::Failed to find CodeQL plugin directory!") | |
| sys.exit(1) | |
| - name: Get CodeQL CLI Cache Data | |
| id: cache_key_gen | |
| env: | |
| CODEQL_PLUGIN_DIR: ${{ steps.find_dir.outputs.codeql_plugin_dir }} | |
| shell: python | |
| run: | | |
| import os | |
| import yaml | |
| codeql_cli_ext_dep_name = 'codeqlcli_windows_ext_dep' | |
| codeql_plugin_file = os.path.join(os.environ['CODEQL_PLUGIN_DIR'], codeql_cli_ext_dep_name + '.yaml') | |
| with open (codeql_plugin_file) as pf: | |
| codeql_cli_ext_dep = yaml.safe_load(pf) | |
| cache_key_name = codeql_cli_ext_dep['name'] | |
| cache_key_version = codeql_cli_ext_dep['version'] | |
| cache_key = f'{cache_key_name}-{cache_key_version}' | |
| codeql_plugin_cli_ext_dep_dir = os.path.join(os.environ['CODEQL_PLUGIN_DIR'], codeql_cli_ext_dep['name'].strip() + '_extdep') | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| print(f'codeql_cli_cache_key={cache_key}', file=fh) | |
| print(f'codeql_cli_ext_dep_dir={codeql_plugin_cli_ext_dep_dir}', file=fh) | |
| - name: Attempt to Load CodeQL CLI From Cache | |
| id: codeqlcli_cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }} | |
| key: ${{ steps.cache_key_gen.outputs.codeql_cli_cache_key }} | |
| - name: Download CodeQL CLI | |
| if: steps.codeqlcli_cache.outputs.cache-hit != 'true' | |
| run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 --codeql | |
| - name: Remove CI Plugins Irrelevant to CodeQL | |
| shell: python | |
| env: | |
| CODEQL_PLUGIN_DIR: ${{ steps.find_dir.outputs.codeql_plugin_dir }} | |
| run: | | |
| import os | |
| import shutil | |
| from pathlib import Path | |
| # Only these two plugins are needed for CodeQL | |
| plugins_to_keep = ['CompilerPlugin'] | |
| plugin_dir = Path('.pytool/Plugin').absolute() | |
| if plugin_dir.is_dir(): | |
| for dir in plugin_dir.iterdir(): | |
| if str(dir.stem) not in plugins_to_keep: | |
| shutil.rmtree(str(dir.absolute()), ignore_errors=True) | |
| - name: CI Build | |
| env: | |
| STUART_CODEQL_PATH: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }} | |
| run: stuart_ci_build -c .pytool/CISettings.py -t DEBUG -p ${{ matrix.Package }} -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 --codeql | |
| - name: Build Cleanup | |
| id: build_cleanup | |
| shell: python | |
| run: | | |
| import os | |
| import shutil | |
| from pathlib import Path | |
| dirs_to_delete = ['ia32', 'x64', 'arm', 'aarch64'] | |
| def delete_dirs(path: Path): | |
| if path.exists() and path.is_dir(): | |
| if path.name.lower() in dirs_to_delete: | |
| print(f'Removed {str(path)}') | |
| shutil.rmtree(path) | |
| return | |
| for child_dir in path.iterdir(): | |
| delete_dirs(child_dir) | |
| build_path = Path(os.environ['GITHUB_WORKSPACE'], 'Build') | |
| delete_dirs(build_path) | |
| - name: Upload Build Logs As An Artifact | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() | |
| with: | |
| name: ${{ matrix.Package }}-${{ steps.convert_arch_hyphen.outputs.arch_list }}-Build-Logs | |
| path: | | |
| **/BUILD_REPORT.TXT | |
| **/OVERRIDELOG.TXT | |
| **/BUILDLOG_*.md | |
| **/BUILDLOG_*.txt | |
| **/CI_*.md | |
| **/CI_*.txt | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Prepare Env Data for CodeQL Upload | |
| id: env_data | |
| env: | |
| PACKAGE_NAME: ${{ matrix.Package }} | |
| shell: python | |
| run: | | |
| import logging | |
| import os | |
| from edk2toollib.utility_functions import RunCmd | |
| from io import StringIO | |
| from pathlib import Path | |
| package = os.environ['PACKAGE_NAME'].strip().lower() | |
| directory_name = 'codeql-analysis-' + package + '-debug' | |
| file_name = 'codeql-db-' + package + '-debug-0.sarif' | |
| sarif_path = Path('Build', directory_name, file_name) | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| if sarif_path.is_file(): | |
| emacs_file_path = sarif_path.with_name(sarif_path.stem + "-emacs.txt") | |
| out_stream_buffer = StringIO() | |
| exit_code = RunCmd("sarif", f"emacs {sarif_path} --output {emacs_file_path} --no-autotrim", | |
| outstream=out_stream_buffer, | |
| logging_level=logging.NOTSET) | |
| print(f'upload_sarif_file=true', file=fh) | |
| print(f'emacs_file_path={emacs_file_path}', file=fh) | |
| print(f'sarif_file_path={sarif_path}', file=fh) | |
| else: | |
| print(f'upload_sarif_file=false', file=fh) | |
| - name: Upload CodeQL Results (SARIF) As An Artifact | |
| uses: actions/upload-artifact@v4 | |
| if: steps.env_data.outputs.upload_sarif_file == 'true' | |
| with: | |
| name: ${{ matrix.Package }}-${{ steps.convert_arch_hyphen.outputs.arch_list }}-CodeQL-SARIF | |
| path: | | |
| ${{ steps.env_data.outputs.emacs_file_path }} | |
| ${{ steps.env_data.outputs.sarif_file_path }} | |
| retention-days: 14 | |
| if-no-files-found: warn | |
| - name: Upload CodeQL Results (SARIF) To GitHub Code Scanning | |
| uses: github/codeql-action/upload-sarif@v3 | |
| if: steps.env_data.outputs.upload_sarif_file == 'true' | |
| with: | |
| # Path to SARIF file relative to the root of the repository. | |
| sarif_file: ${{ steps.env_data.outputs.sarif_file_path }} | |
| # Optional category for the results. Used to differentiate multiple results for one commit. | |
| # Each package is a separate category. | |
| category: ${{ matrix.Package }} |