feat: add enhanced search

docs/tutorials/gcp/organization.md

@@ -1,9 +1,12 @@
 # GCP Organization
 
-By default, Prowler will scan all the Google Cloud projects that the authenticated user has access to.
+By default, Prowler scans all Google Cloud projects accessible to the authenticated user.
 
-If you want to scan only projects from a specific organization, you can use the `--organization-id` argument.
+To limit the scan to projects within a specific Google Cloud organization, use the `--organization-id` option with the GCP organization ID:
 
 ```console
 prowler gcp --organization-id organization-id
 ```
+
+???+ note
+    With this option, Prowler retrieves all projects within the specified organization, including those organized in folders and nested subfolders. This ensures that every project under the organization’s hierarchy is scanned, providing full visibility across the entire organization.

prowler/providers/gcp/exceptions/exceptions.py

@@ -42,6 +42,10 @@ class GCPBaseException(ProwlerException):
             "message": "Provider does not match with the expected project_id",
             "remediation": "Check the provider and ensure it matches the expected project_id.",
         },
+        (3009, "GCPCloudAssetAPINotUsedError"): {
+            "message": "Cloud Asset API not used",
+            "remediation": "Enable the Cloud Asset API for the project.",
+        },
     }
 
     def __init__(self, code, file=None, original_exception=None, message=None):
@@ -126,3 +130,10 @@ def __init__(self, file=None, original_exception=None, message=None):
         super().__init__(
             3008, file=file, original_exception=original_exception, message=message
         )
+
+
+class GCPCloudAssetAPINotUsedError(GCPBaseException):
+    def __init__(self, file=None, original_exception=None, message=None):
+        super().__init__(
+            3009, file=file, original_exception=original_exception, message=message
+        )

prowler/providers/gcp/gcp_provider.py

@@ -20,6 +20,7 @@
 from prowler.providers.common.models import Audit_Metadata, Connection
 from prowler.providers.common.provider import Provider
 from prowler.providers.gcp.exceptions.exceptions import (
+    GCPCloudAssetAPINotUsedError,
     GCPCloudResourceManagerAPINotUsedError,
     GCPGetProjectError,
     GCPHTTPError,
@@ -444,47 +445,87 @@ def get_projects(
         try:
             projects = {}
 
-            service = discovery.build(
-                "cloudresourcemanager", "v1", credentials=credentials
-            )
             if organization_id:
-                request = service.projects().list(
-                    filter=f'parent.type:"organization" parent.id:"{organization_id}"'
+                # Initialize Cloud Asset Inventory API for recursive project retrieval
+                asset_service = discovery.build(
+                    "cloudasset", "v1", credentials=credentials
+                )
+                # Set the scope to the specified organization and filter for projects
+                scope = f"organizations/{organization_id}"
+                request = asset_service.assets().list(
+                    parent=scope,
+                    assetTypes=["cloudresourcemanager.googleapis.com/Project"],
+                    contentType="RESOURCE",
                 )
-            else:
-                request = service.projects().list()
-
-            while request is not None:
-                response = request.execute()
-
-                for project in response.get("projects", []):
-                    labels = {}
-                    for key, value in project.get("labels", {}).items():
-                        labels[key] = value
-
-                    project_id = project["projectId"]
-                    gcp_project = GCPProject(
-                        number=project["projectNumber"],
-                        id=project_id,
-                        name=project.get("name", project_id),
-                        lifecycle_state=project["lifecycleState"],
-                        labels=labels,
-                    )
 
-                    if (
-                        "parent" in project
-                        and "type" in project["parent"]
-                        and project["parent"]["type"] == "organization"
-                    ):
-                        organization_id = project["parent"]["id"]
+                while request is not None:
+                    response = request.execute()
+
+                    for asset in response.get("assets", []):
+                        # Extract labels and other project details
+                        labels = {
+                            k: v
+                            for k, v in asset["resource"]["data"]
+                            .get("labels", {})
+                            .items()
+                        }
+                        project_id = asset["resource"]["data"]["projectId"]
+                        gcp_project = GCPProject(
+                            number=asset["resource"]["data"]["projectNumber"],
+                            id=project_id,
+                            name=asset["resource"]["data"].get("name", project_id),
+                            lifecycle_state=asset["resource"]["data"].get(
+                                "lifecycleState"
+                            ),
+                            labels=labels,
+                        )
                         gcp_project.organization = GCPOrganization(
                             id=organization_id, name=f"organizations/{organization_id}"
                         )
 
-                    projects[project_id] = gcp_project
-                request = service.projects().list_next(
-                    previous_request=request, previous_response=response
+                        projects[project_id] = gcp_project
+
+                    request = asset_service.assets().list_next(
+                        previous_request=request, previous_response=response
+                    )
+
+            else:
+                # Initialize Cloud Resource Manager API for simple project listing
+                service = discovery.build(
+                    "cloudresourcemanager", "v1", credentials=credentials
                 )
+                request = service.projects().list()
+
+                while request is not None:
+                    response = request.execute()
+
+                    for project in response.get("projects", []):
+                        # Extract labels and other project details
+                        labels = {k: v for k, v in project.get("labels", {}).items()}
+                        project_id = project["projectId"]
+                        gcp_project = GCPProject(
+                            number=project["projectNumber"],
+                            id=project_id,
+                            name=project.get("name", project_id),
+                            lifecycle_state=project["lifecycleState"],
+                            labels=labels,
+                        )
+
+                        # Set organization if present in the project metadata
+                        if (
+                            "parent" in project
+                            and project["parent"].get("type") == "organization"
+                        ):
+                            parent_org_id = project["parent"]["id"]
+                            gcp_project.organization = GCPOrganization(
+                                id=parent_org_id, name=f"organizations/{parent_org_id}"
+                            )
+
+                        projects[project_id] = gcp_project
+
+                    request = service.projects().list_next(
+                        previous_request=request, previous_response=response
+                    )
 
         except HttpError as http_error:
             if "Cloud Resource Manager API has not been used" in str(http_error):
@@ -494,6 +535,13 @@ def get_projects(
                 raise GCPCloudResourceManagerAPINotUsedError(
                     file=__file__, original_exception=http_error
                 )
+            elif "Cloud Asset API has not been used" in str(http_error):
+                logger.critical(
+                    "Cloud Asset API has not been used before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/cloudasset.googleapis.com/ then retry."
+                )
+                raise GCPCloudAssetAPINotUsedError(
+                    file=__file__, original_exception=http_error
+                )
             else:
                 logger.error(
                     f"{http_error.__class__.__name__}[{http_error.__traceback__.tb_lineno}]: {http_error}"

tests/providers/gcp/gcp_provider_test.py

@@ -17,7 +17,7 @@
     GCPTestConnectionError,
 )
 from prowler.providers.gcp.gcp_provider import GcpProvider
-from prowler.providers.gcp.models import GCPIdentityInfo, GCPProject
+from prowler.providers.gcp.models import GCPIdentityInfo, GCPOrganization, GCPProject
 
 
 class TestGCPProvider:
@@ -87,6 +87,7 @@ def test_is_project_matching(self):
         arguments = Namespace()
         arguments.project_id = []
         arguments.excluded_project_id = []
+        arguments.organization_id = None
         arguments.list_project_id = False
         arguments.credentials_file = ""
         arguments.impersonate_service_account = ""
@@ -131,6 +132,7 @@ def test_is_project_matching(self):
             return_value=mocked_service,
         ):
             gcp_provider = GcpProvider(
+                arguments.organization_id,
                 arguments.project_id,
                 arguments.excluded_project_id,
                 arguments.credentials_file,
@@ -168,6 +170,7 @@ def test_setup_session_with_credentials_file_no_impersonate(self):
         arguments = Namespace()
         arguments.project_id = []
         arguments.excluded_project_id = []
+        arguments.organization_id = None
         arguments.list_project_id = False
         arguments.credentials_file = "test_credentials_file"
         arguments.impersonate_service_account = ""
@@ -206,6 +209,7 @@ def test_setup_session_with_credentials_file_no_impersonate(self):
             return_value=mocked_service,
         ):
             gcp_provider = GcpProvider(
+                arguments.organization_id,
                 arguments.project_id,
                 arguments.excluded_project_id,
                 arguments.credentials_file,
@@ -230,6 +234,7 @@ def test_setup_session_with_credentials_file_and_impersonate(self):
         arguments = Namespace()
         arguments.project_id = []
         arguments.excluded_project_id = []
+        arguments.organization_id = None
         arguments.list_project_id = False
         arguments.credentials_file = "test_credentials_file"
         arguments.impersonate_service_account = "test-impersonate-service-account"
@@ -268,6 +273,7 @@ def test_setup_session_with_credentials_file_and_impersonate(self):
             return_value=mocked_service,
         ):
             gcp_provider = GcpProvider(
+                arguments.organization_id,
                 arguments.project_id,
                 arguments.excluded_project_id,
                 arguments.credentials_file,
@@ -291,6 +297,78 @@ def test_setup_session_with_credentials_file_and_impersonate(self):
                 == "test-impersonate-service-account"
             )
 
+    def test_setup_session_with_organization_id(self):
+        mocked_credentials = MagicMock()
+
+        mocked_credentials.refresh.return_value = None
+        mocked_credentials._service_account_email = "test-service-account-email"
+
+        arguments = Namespace()
+        arguments.project_id = []
+        arguments.excluded_project_id = []
+        arguments.organization_id = "test-organization-id"
+        arguments.list_project_id = False
+        arguments.credentials_file = "test_credentials_file"
+        arguments.impersonate_service_account = ""
+        arguments.config_file = default_config_file_path
+        arguments.fixer_config = default_fixer_config_file_path
+
+        projects = {
+            "test-project": GCPProject(
+                number="55555555",
+                id="project/55555555",
+                name="test-project",
+                labels={"test": "value"},
+                lifecycle_state="",
+                organization=GCPOrganization(
+                    id="test-organization-id",
+                    name="test-organization",
+                    display_name="Test Organization",
+                ),
+            )
+        }
+
+        mocked_service = MagicMock()
+
+        mocked_service.projects.list.return_value = MagicMock(
+            execute=MagicMock(return_value={"projects": projects})
+        )
+        with patch(
+            "prowler.providers.gcp.gcp_provider.GcpProvider.get_projects",
+            return_value=projects,
+        ), patch(
+            "prowler.providers.gcp.gcp_provider.GcpProvider.update_projects_with_organizations",
+            return_value=None,
+        ), patch(
+            "os.path.abspath",
+            return_value="test_credentials_file",
+        ), patch(
+            "prowler.providers.gcp.gcp_provider.default",
+            return_value=(mocked_credentials, MagicMock()),
+        ), patch(
+            "prowler.providers.gcp.gcp_provider.discovery.build",
+            return_value=mocked_service,
+        ):
+            gcp_provider = GcpProvider(
+                arguments.organization_id,
+                arguments.project_id,
+                arguments.excluded_project_id,
+                arguments.credentials_file,
+                arguments.impersonate_service_account,
+                arguments.list_project_id,
+                arguments.config_file,
+                arguments.fixer_config,
+                client_id=None,
+                client_secret=None,
+                refresh_token=None,
+            )
+            assert environ["GOOGLE_APPLICATION_CREDENTIALS"] == "test_credentials_file"
+            assert gcp_provider.session is not None
+            assert (
+                gcp_provider.projects["test-project"].organization.id
+                == "test-organization-id"
+            )
+
     def test_print_credentials_default_options(self, capsys):
         mocked_credentials = MagicMock()
 
@@ -300,6 +378,7 @@ def test_print_credentials_default_options(self, capsys):
         arguments = Namespace()
         arguments.project_id = []
         arguments.excluded_project_id = []
+        arguments.organization_id = None
         arguments.list_project_id = False
         arguments.credentials_file = "test_credentials_file"
         arguments.impersonate_service_account = ""
@@ -338,6 +417,7 @@ def test_print_credentials_default_options(self, capsys):
             return_value=mocked_service,
         ):
             gcp_provider = GcpProvider(
+                arguments.organization_id,
                 arguments.project_id,
                 arguments.excluded_project_id,
                 arguments.credentials_file,
@@ -369,6 +449,7 @@ def test_print_credentials_impersonated_service_account(self, capsys):
         arguments = Namespace()
         arguments.project_id = []
         arguments.excluded_project_id = []
+        arguments.organization_id = None
         arguments.list_project_id = False
         arguments.credentials_file = "test_credentials_file"
         arguments.impersonate_service_account = "test-impersonate-service-account"
@@ -407,6 +488,7 @@ def test_print_credentials_impersonated_service_account(self, capsys):
             return_value=mocked_service,
         ):
             gcp_provider = GcpProvider(
+                arguments.organization_id,
                 arguments.project_id,
                 arguments.excluded_project_id,
                 arguments.credentials_file,
@@ -438,6 +520,7 @@ def test_print_credentials_excluded_project_ids(self, capsys):
         arguments = Namespace()
         arguments.project_id = []
         arguments.excluded_project_id = ["test-excluded-project"]
+        arguments.organization_id = None
         arguments.list_project_id = False
         arguments.credentials_file = "test_credentials_file"
         arguments.impersonate_service_account = ""
@@ -484,6 +567,7 @@ def test_print_credentials_excluded_project_ids(self, capsys):
             return_value=mocked_service,
         ):
             gcp_provider = GcpProvider(
+                arguments.organization_id,
                 arguments.project_id,
                 arguments.excluded_project_id,
                 arguments.credentials_file,