Merge branch 'main' into mv/level

fixbackend/all_models.py

@@ -30,3 +30,4 @@
 from fixbackend.notification.notification_provider_config_repo import NotificationProviderConfigEntity # noqa
 from fixbackend.notification.user_notification_repo import UserNotificationSettingsEntity # noqa
 from fixbackend.permissions.role_repository import UserRoleAssignmentEntity # noqa
+from fixbackend.notification.email.scheduled_email import ScheduledEmailEntity, ScheduledEmailSentEntity # noqa

fixbackend/app.py

@@ -12,8 +12,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-import logging
 import base64
+import logging
 import os
 from contextlib import asynccontextmanager
 from dataclasses import replace
@@ -93,6 +93,7 @@
 )
 from fixbackend.metering.metering_repository import MeteringRepository
 from fixbackend.middleware.x_real_ip import RealIpMiddleware
+from fixbackend.notification.email.scheduled_email import ScheduledEmailSender
 from fixbackend.notification.notification_router import notification_router, unsubscribe_router
 from fixbackend.notification.notification_service import NotificationService
 from fixbackend.permissions.role_repository import RoleRepositoryImpl
@@ -405,6 +406,7 @@ async def setup_teardown_dispatcher(_: FastAPI) -> AsyncIterator[None]:
  workspace_repo,
  ),
  )
+ deps.add(SN.scheduled_email_sender, ScheduledEmailSender(notification_service.email_sender, session_maker))
 
  async with deps:
  log.info("Application services started.")

fixbackend/auth/models/orm.py

@@ -20,8 +20,9 @@
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from fixbackend.auth import models
-from fixbackend.base_model import Base
+from fixbackend.base_model import Base, CreatedUpdatedMixin
 from fixbackend.ids import UserId
+
 from fixbackend.permissions.role_repository import UserRoleAssignmentEntity
 from fixbackend.sqlalechemy_extensions import GUID
 
@@ -61,13 +62,10 @@ class UserMFARecoveryCode(Base):
  code_hash: Mapped[str] = mapped_column(String(length=64), primary_key=True)
 
 
-class User(SQLAlchemyBaseUserTableUUID, Base):
+class User(SQLAlchemyBaseUserTableUUID, CreatedUpdatedMixin, Base):
  otp_secret: Mapped[Optional[str]] = mapped_column(String(length=64), nullable=True)
  is_mfa_active: Mapped[bool] = mapped_column(Boolean, default=False, nullable=True)
  oauth_accounts: Mapped[List[OAuthAccount]] = relationship("OAuthAccount", lazy="joined")
- mfa_recovery_codes: Mapped[List[UserMFARecoveryCode]] = relationship(
- "UserMFARecoveryCode", backref="user", lazy="joined"
- )
  roles: Mapped[List[UserRoleAssignmentEntity]] = relationship(
  "UserRoleAssignmentEntity", backref="user", lazy="joined"
  )

fixbackend/dependencies.py

@@ -63,6 +63,7 @@ class ServiceNames:
  billing_entry_service = "billing_entry_services"
  role_repository = "role_repository"
  jwt_service = "jwt_service"
+ scheduled_email_sender = "scheduled_email_sender"
 
 
 class FixDependencies(Dependencies):

fixbackend/graph_db/models.py

@@ -24,7 +24,7 @@
 from fixbackend.ids import WorkspaceId
 
 
-@define
+@define(hash=True, slots=True, frozen=True)
 class GraphDatabaseAccess:
  workspace_id: WorkspaceId
  server: str

fixbackend/inventory/inventory_service.py

@@ -102,6 +102,7 @@
 ReportSeverityPriority: Dict[str, int] = defaultdict(lambda: 0, **{n: idx for idx, n in enumerate(ReportSeverityList)})
 ReportSeverityIncluded: Dict[str, List[str]] = {n: ReportSeverityList[idx:] for idx, n in enumerate(ReportSeverityList)}
 Inventory = "inventory-service"
+DecoratedFn = TypeVar("DecoratedFn", bound=Callable[..., Any])
 
 
 def dict_values_by(d: Mapping[T, Iterable[V]], fn: Callable[[T], Any]) -> Iterable[V]:
@@ -206,6 +207,48 @@ async def evict_cache(self, workspace_id: WorkspaceId) -> None:
  # evict the cache for the tenant in the cluster
  await self.cache.evict(str(workspace_id))
 
+ async def checks(
+ self,
+ db: GraphDatabaseAccess,
+ provider: Optional[str] = None,
+ service: Optional[str] = None,
+ category: Optional[str] = None,
+ kind: Optional[str] = None,
+ check_ids: Optional[List[str]] = None,
+ ids_only: Optional[bool] = None,
+ ) -> List[Json]:
+ async def fetch_checks(*_: Any) -> List[Json]:
+ return await self.client.checks(
+ db,
+ provider=provider,
+ service=service,
+ category=category,
+ kind=kind,
+ check_ids=check_ids,
+ ids_only=ids_only,
+ )
+
+ return await self.cache.call(fetch_checks, key=str(db.workspace_id))(
+ provider, service, category, kind, check_ids, ids_only # parameters passed as cache key
+ )
+
+ async def benchmarks(
+ self,
+ db: GraphDatabaseAccess,
+ benchmarks: Optional[List[str]] = None,
+ short: Optional[bool] = None,
+ with_checks: Optional[bool] = None,
+ ids_only: Optional[bool] = None,
+ ) -> List[Json]:
+ async def fetch_benchmarks(*_: Any) -> List[Json]:
+ return await self.client.benchmarks(
+ db, benchmarks=benchmarks, short=short, with_checks=with_checks, ids_only=ids_only
+ )
+
+ return await self.cache.call(fetch_benchmarks, key=str(db.workspace_id))(
+ benchmarks, short, with_checks, ids_only # parameters passed as cache key
+ )
+
  async def report_info(self, db: GraphDatabaseAccess) -> Json:
  async def compute_report_info() -> Json:
  benchmark_ids, check_ids = await asyncio.gather(

fixbackend/inventory/router.py

@@ -68,6 +68,18 @@ async def update_report_config(graph_db: CurrentGraphDbDependency, config: Repor
  async def report_info(graph_db: CurrentGraphDbDependency) -> Json:
  return await inventory().report_info(graph_db)
 
+ @router.get("/report/benchmarks", tags=["report-management"])
+ async def list_benchmarks(
+ graph_db: CurrentGraphDbDependency,
+ benchmarks: Optional[List[str]] = None,
+ short: Optional[bool] = None,
+ with_checks: Optional[bool] = None,
+ ids_only: Optional[bool] = None,
+ ) -> List[Json]:
+ return await inventory().benchmarks(
+ graph_db, benchmarks=benchmarks, short=short, with_checks=with_checks, ids_only=ids_only
+ )
+
  @router.get("/report/benchmark/{benchmark_name}", tags=["report-management"])
  async def get_benchmark(benchmark_name: str, graph_db: CurrentGraphDbDependency) -> Json:
  return await inventory().client.call_json(graph_db, "get", f"/report/benchmark/{benchmark_name}")
@@ -83,6 +95,26 @@ async def delete_benchmark(benchmark_name: str, graph_db: CurrentGraphDbDependen
  )
  return Response(status_code=204)
 
+ @router.get("/report/checks", tags=["report-management"])
+ async def list_checks(
+ graph_db: CurrentGraphDbDependency,
+ provider: Optional[str] = None,
+ service: Optional[str] = None,
+ category: Optional[str] = None,
+ kind: Optional[str] = None,
+ check_ids: Optional[List[str]] = None,
+ ids_only: Optional[bool] = None,
+ ) -> List[Json]:
+ return await inventory().checks(
+ graph_db,
+ provider=provider,
+ service=service,
+ category=category,
+ kind=kind,
+ check_ids=check_ids,
+ ids_only=ids_only,
+ )
+
  @router.get("/report/check/{check_id}", tags=["report-management"])
  async def get_check(check_id: str, graph_db: CurrentGraphDbDependency) -> Json:
  return await inventory().client.call_json(graph_db, "get", f"/report/check/{check_id}")

fixbackend/notification/email/email_messages.py

@@ -21,10 +21,12 @@
 
 from fixbackend.ids import CloudAccountId, WorkspaceId
 
+TemplatesPath = Path(__file__).parent / "templates"
+
 
 @lru_cache(maxsize=1)
 def get_env() -> Environment:
- return Environment(loader=FileSystemLoader(Path(__file__).parent / "templates"), undefined=StrictUndefined)
+ return Environment(loader=FileSystemLoader(TemplatesPath), undefined=StrictUndefined)
 
 
 def render(template_name: str, **kwargs: Any) -> str:

fixbackend/notification/email/scheduled_email.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2024. Some Engineering
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import logging
+from datetime import datetime, timedelta
+
+from fastapi_users_db_sqlalchemy.generics import GUID
+from fixcloudutils.asyncio.periodic import Periodic
+from fixcloudutils.service import Service
+from fixcloudutils.util import utc
+from sqlalchemy import String, Integer, select, Index, and_, func, text
+from sqlalchemy.orm import Mapped, mapped_column
+
+from fixbackend.auth.models.orm import User
+from fixbackend.base_model import Base
+from fixbackend.ids import UserId
+from fixbackend.notification.email import email_messages
+from fixbackend.notification.email.email_sender import EmailSender
+from fixbackend.sqlalechemy_extensions import UTCDateTime
+from fixbackend.types import AsyncSessionMaker
+from fixbackend.utils import uid
+
+log = logging.getLogger(__name__)
+
+
+class ScheduledEmailEntity(Base):
+ __tablename__ = "scheduled_email"
+ id: Mapped[GUID] = mapped_column(GUID, primary_key=True)
+ kind: Mapped[str] = mapped_column(String(64), nullable=False)
+ after: Mapped[int] = mapped_column(Integer, nullable=False)
+
+
+class ScheduledEmailSentEntity(Base):
+ __tablename__ = "scheduled_email_sent"
+ id: Mapped[GUID] = mapped_column(GUID, primary_key=True)
+ user_id: Mapped[UserId] = mapped_column(GUID, nullable=False)
+ kind: Mapped[str] = mapped_column(String(64), nullable=False)
+ at: Mapped[datetime] = mapped_column(UTCDateTime, nullable=False)
+
+ user_kind_index = Index("user_kind_index", "user_id", "kind")
+
+
+class ScheduledEmailSender(Service):
+ def __init__(self, email_sender: EmailSender, session_maker: AsyncSessionMaker) -> None:
+ self.email_sender = email_sender
+ self.session_maker = session_maker
+ self.periodic = Periodic("scheduled_email_sender", self._send_emails, timedelta(seconds=600))
+
+ async def start(self) -> None:
+ await self.periodic.start()
+
+ async def stop(self) -> None:
+ await self.periodic.stop()
+
+ async def _send_emails(self) -> None:
+ async with self.session_maker() as session:
+ stmt = (
+ select(User, ScheduledEmailEntity)
+ .select_from(
+ # This uses a literal TRUE to simulate a cross join
+ User.__table__.join(ScheduledEmailEntity.__table__, text("true"))
+ )
+ .outerjoin(
+ ScheduledEmailSentEntity,
+ and_(
+ User.id == ScheduledEmailSentEntity.user_id, # type: ignore
+ ScheduledEmailEntity.kind == ScheduledEmailSentEntity.kind,
+ ),
+ )
+ .where(
+ and_(
+ text("user.created_at + INTERVAL scheduled_email.after SECOND") < func.now(),
+ ScheduledEmailSentEntity.id.is_(None),
+ )
+ )
+ )
+ result = await session.execute(stmt)
+ user: User
+ to_send: ScheduledEmailEntity
+ for user, to_send in result.unique().all():
+ subject = email_messages.render(f"{to_send.kind}.subject").strip()
+ txt = email_messages.render(f"{to_send.kind}.txt")
+ html = email_messages.render(f"{to_send.kind}.html")
+ log.info(f"Sending email to {user.email} with subject {subject} and body {html}")
+ await self.email_sender.send_email(to=user.email, subject=subject, text=txt, html=html)
+ # mark this kind of email as sent
+ session.add(ScheduledEmailSentEntity(id=uid(), user_id=user.id, kind=to_send.kind, at=utc()))
+ await session.commit()

fixbackend/notification/email/templates/day1.html

@@ -0,0 +1,84 @@
+{% extends "base.html" %}
+
+{% block content %}
+
+ <h1> Day 1: Effortless Search, Powerful Insights </h1>
+
+ <p>
+ Imagine having the ability to pinpoint any resource in your cloud environment within seconds.
+ With Fix, that's not just possible; it's your new reality.
+ Our search functionality is designed to streamline your workflows, enhance security, and ensure you're always just a
+ few keystrokes away from finding exactly what you need.
+ </p>
+
+ <p>
+ This tutorial assumes that you connected at least one cloud account with Fix. If not already happened, please visit
+ the setup cloud accounts page in fix and connect your accounts.
+ </p>
+
+ <h2>Filter by Origin</h2>
+ <p>
+ Log into your Fix account and navigate to the Inventory.
+ Here, you find dropdowns for the most common filters.
+ </p>
+ <p>
+ <img src="https://cdn.some.engineering/fix/assets/101/day1/filter-overview.png" width="530" alt="Filer Overview"/>
+ </p>
+ <p>
+ Filter resources by cloud, account, region or zone is straightforward.
+ You can also narrow down the list of resources by a specific tag.
+ </p>
+
+ <h2>Filter by Security Severity</h2>
+ <p>
+ Fix analyzes all your resources against different security benchmarks and compliance frameworks.
+ Fix marks all security vulnerabilities on the resource.
+ The severity filter now allows you to filter for resources that have vulnerabilities of a specific severity.
+ </p>
+
+ <p>
+ <img src="https://cdn.some.engineering/fix/assets/101/day1/severity-filter.png" width="280" alt="Severity Filter"/>
+ </p>
+
+ <h2>Filter by Kind</h2>
+ <p>
+ The Kinds filter allows you to filter resources of a specific service, e.g. Lambda Functions or IAM roles. This is
+ the goto filter if you want to narrow down specific resources by specific property filters.
+ </p>
+ <p>
+ <img src="https://cdn.some.engineering/fix/assets/101/day1/kinds-filter.png" width="280" alt="Kinds Filter"/>
+ </p>
+
+ <h2>Filter by Property</h2>
+ <p>
+ Every resource kind has its own set of properties. Fix allows you to filter resources by any property. Press the +
+ button. Based on the selected kind of resource, you now see all possible properties.
+ </p>
+ <p>
+ <img src="https://cdn.some.engineering/fix/assets/101/day1/property-filter.png" width="250" alt="Property Filter"/>
+ </p>
+
+ <p>
+ Once the property is selected, you can define the operation and filter value. The value dropdown shows all possible
+ values that exist for the selected property in the current search configuration.
+ </p>
+
+ <p>
+ <img src="https://cdn.some.engineering/fix/assets/101/day1/property-value.png" width="480" alt="Property Value"/>
+ </p>
+
+ <p>
+ We're Here to Help
+ If any steps feel unclear, or if you encounter any bumps along the road, our team is standing by.
+ </p>
+ <p>
+ Contact us at <a href="mailto:[email protected]">[email protected]</a> or ping us on <a
+ href="https://discord.gg/fixsecurity">Discord</a>.
+ </p>
+
+ Happy searching,<br/>
+ The Fix Team
+
+
+
+{% endblock content %}

fixbackend/notification/email/templates/day1.subject

@@ -0,0 +1 @@
+Fix 101 Day 1: Effortless Search, Powerful Insights