Merge pull request #4850 from systeminit/jkeiser/lago-upload-metrics

Billing lambda: log upload stats using INFO

component/lambda/functions/si_logger.py

@@ -2,4 +2,4 @@
 
 
 logger = logging.getLogger()
-logging.basicConfig(level=logging.INFO)
+logger.setLevel("INFO")

component/lambda/functions/si_redshift.py

@@ -17,6 +17,7 @@
 from botocore.client import Config
 import time
 from si_logger import logger
+import logging
 
 
 class DatabaseConnectParams(TypedDict):
@@ -54,12 +55,15 @@ def from_env(
  def __init__(
  self,
  session: boto3.Session,
+ *,
  wait_interval_seconds: float = 0.25,
+ report_interval_seconds: float = 5,
  **database_params: Unpack[DatabaseConnectParams],
  ):
  self._session = session
  self._database_params = database_params
  self._wait_interval_seconds = wait_interval_seconds
+ self._report_interval_seconds = report_interval_seconds
  self._client = self._connect()
 
  def query(self, Sql: str, **Parameters: object):
@@ -94,6 +98,8 @@ def execute(self, Sql: str, **Parameters: object):
  )
  )
 
+ last_report = time.time()
+
  while True:
  response = self.with_client(
  lambda client: client.describe_statement(Id=statement["Id"])
@@ -110,9 +116,11 @@ def execute(self, Sql: str, **Parameters: object):
  case "ABORTED":
  raise Exception(f"Query aborted (Id={statement['Id']})")
 
- logger.debug(
- f"Query status: {status}. Waiting {self._wait_interval_seconds}s for completion... (Id={statement['Id']})"
- )
+ if time.time() - last_report >= self._report_interval_seconds:
+ last_report = time.time()
+ logger.log(logging.INFO,
+ f"Query status: {status}. Waiting {self._wait_interval_seconds}s for completion... (Id={statement['Id']})"
+ )
 
  time.sleep(self._wait_interval_seconds)
 

component/lambda/functions/upload_billing_resource_hours_to_lago.py

@@ -104,7 +104,7 @@ def lambda_handler(lambda_event={}, _context=None):
  last_hour_end = None
  break
 
- logger.warning(
+ logger.info(
  f"Uploaded {uploaded_events} events in {batch_hours}-hour batch starting {-first_hour_start} hours ago!"
  )
  last_hour_end = first_hour_start if uploaded_events > 0 else None