BDOG-3253 refactor add configurable metrics for Unsafe Content detection

app/uk/gov/hmrc/servicemetrics/config/Module.scala → app/uk/gov/hmrc/servicemetrics/Module.scala

@@ -14,14 +14,16 @@
  * limitations under the License.
  */
 
-package uk.gov.hmrc.servicemetrics.config
+package uk.gov.hmrc.servicemetrics
 
-import com.google.inject.AbstractModule
-import uk.gov.hmrc.servicemetrics.scheduler.{MongoMetricsScheduler, MongoNotificationsScheduler}
+import play.api.inject.Binding
+import play.api.{Configuration, Environment}
+import uk.gov.hmrc.servicemetrics.scheduler.{MetricsScheduler, NotificationsScheduler}
 
-class Module extends AbstractModule:
+class Module extends play.api.inject.Module:
 
- override def configure(): Unit =
- bind(classOf[AppConfig ]).asEagerSingleton()
- bind(classOf[MongoMetricsScheduler ]).asEagerSingleton()
- bind(classOf[MongoNotificationsScheduler]).asEagerSingleton()
+ override def bindings(environment: Environment, configuration: Configuration): Seq[Binding[_]] =
+ Seq(
+ bind[MetricsScheduler ].toSelf.eagerly()
+ , bind[NotificationsScheduler].toSelf.eagerly()
+ )

app/uk/gov/hmrc/servicemetrics/binders/Binders.scala

@@ -16,18 +16,36 @@
 
 package uk.gov.hmrc.servicemetrics.binders
 
-import play.api.mvc.QueryStringBindable
+import play.api.mvc.{PathBindable, QueryStringBindable}
 
 import java.time.Instant
 import scala.util.Try
 
 object Binders:
 
- implicit def instantBindable(implicit strBinder: QueryStringBindable[String]): QueryStringBindable[Instant] =
- new QueryStringBindable[Instant]:
- override def bind(key: String, params: Map[String, Seq[String]]): Option[Either[String, Instant]] =
-  strBinder.bind(key, params)
-  .map(_.flatMap(s => Try(Instant.parse(s)).toEither.left.map(_.getMessage)))
+ given QueryStringBindable[Instant] =
+ queryStringBindableFromString[Instant](
+ s => Some(Try(Instant.parse(s)).toEither.left.map(_.getMessage)),
+ _.toString
+ )
 
- override def unbind(key: String, value: Instant): String =
- strBinder.unbind(key, value.toString)
+ def queryStringBindableFromString[T](parse: String => Option[Either[String, T]], asString: T => String)(using strBinder: QueryStringBindable[String]): QueryStringBindable[T] =
+ new QueryStringBindable[T]:
+ override def bind(key: String, params: Map[String, Seq[String]]): Option[Either[String, T]] =
+ strBinder.bind(key, params) match
+ case Some(Right(s)) if s.trim.nonEmpty => parse(s.trim)
+ case _ => None
+
+ override def unbind(key: String, value: T): String =
+ strBinder.unbind(key, asString(value))
+
+ /** `summon[PathBindable[String]].transform` doesn't allow us to provide failures.
+ * This function provides `andThen` semantics
+ */
+ def pathBindableFromString[T](parse: String => Either[String, T], asString: T => String)(using strBinder: PathBindable[String]): PathBindable[T] =
+ new PathBindable[T]:
+ override def bind(key: String, value: String): Either[String, T] =
+ parse(value)
+
+ override def unbind(key: String, value: T): String =
+ asString(value)

app/uk/gov/hmrc/servicemetrics/config/AppConfig.scala

@@ -18,9 +18,92 @@ package uk.gov.hmrc.servicemetrics.config
 
 import javax.inject.{Inject, Singleton}
 import play.api.Configuration
+import java.net.URLEncoder
 
 @Singleton
 class AppConfig @Inject()(config: Configuration):
 
  val collectionSizesHistoryFrequencyDays: Int =
  config.get[Int]("mongo-collection-size-history.frequency.days")
+
+ private val longRunningQueryInMilliseconds: Int =
+ config.get[Int]("microservice.services.elasticsearch.long-running-query-in-milliseconds")
+
+ import AppConfig.{LogMetric, LogMetricId, LogConfigType}
+ val logMetrics: List[LogMetric] =
+ LogMetric(LogMetricId.SlowRunningQuery, "Slow Running Query", LogConfigType.AverageMongoDuration(s"duration:>${longRunningQueryInMilliseconds}"), config.get[String]("alerts.slack.kibana.links.slow-running-query")) ::
+ LogMetric(LogMetricId.NonIndexedQuery , "Non-indexed Query" , LogConfigType.AverageMongoDuration("scan:COLLSCAN") , config.get[String]("alerts.slack.kibana.links.non-indexed-query") ) ::
+ LogMetric(LogMetricId.UnsafeContent , "Unsafe Content" , LogConfigType.GenericSearch("tags.raw:\\\"UnsafeContent\\\"") , config.get[String]("alerts.slack.kibana.links.unsafe-content") ) ::
+ Nil
+
+ def findLogMetricById(logMetricId: LogMetricId): LogMetric =
+ logMetrics.find(_.id == logMetricId) match
+ case None => sys.error(s"Configuration issue - could not find ${logMetricId.asString}")
+ case Some(x) => x
+
+ import uk.gov.hmrc.servicemetrics.persistence.LogHistoryRepository
+ def createMessage(team: String, logs: Seq[LogHistoryRepository.LogHistory]): Seq[String] =
+ logs
+ .groupBy(_.logType)
+ .foldLeft(Seq( s"Hi *$team*, PlatOps would like to notify you about the following Kibana logs:")):
+ case (acc, (logType: LogHistoryRepository.LogType.GenericSearch, ns)) =>
+ acc :+
+ ns.sortBy(x => (x.service, x.environment))
+ .map:
+ case n =>
+ val logMetric = findLogMetricById(logType.logMetricId)
+ val link = logMetric
+ .kibanaLink
+ .replace(s"$${env}" , URLEncoder.encode(n.environment.asString, "UTF-8"))
+ .replace(s"$${service}", URLEncoder.encode(n.service , "UTF-8"))
+ s"• service *${n.service}* in *${n.environment.asString}* has ${logMetric.displayName} - <$link|see kibana>"
+ .distinct
+ .mkString("\n")
+ case (acc, (logType: LogHistoryRepository.LogType.AverageMongoDuration, ns)) =>
+ acc :+
+ ns.sortBy(x => (x.service, x.environment))
+ .flatMap:
+ case n => n.logType.asInstanceOf[LogHistoryRepository.LogType.AverageMongoDuration].details.map(detail => (n, detail)) // TODO :-(
+ .map:
+ case (n, detail) =>
+ val logMetric = findLogMetricById(logType.logMetricId)
+ val link = logMetric
+ .kibanaLink
+ .replace(s"$${env}" , URLEncoder.encode(n.environment.asString, "UTF-8"))
+ .replace(s"$${database}", URLEncoder.encode(detail.database , "UTF-8"))
+ s"• service *${n.service}* in *${n.environment.asString}* has ${logMetric.displayName} - <$link|see kibana>"
+ .distinct
+ .mkString("\n")
+
+
+object AppConfig:
+ import play.api.libs.json.{Reads, Writes}
+ import play.api.mvc.{PathBindable, QueryStringBindable}
+ import uk.gov.hmrc.servicemetrics.util.{FromString, FromStringEnum, Parser}
+
+ import FromStringEnum._
+
+ given Parser[LogMetricId] = Parser.parser(LogMetricId.values)
+
+ enum LogMetricId(
+ override val asString: String
+ ) extends FromString
+ derives Ordering, Reads, Writes, PathBindable, QueryStringBindable:
+ case SlowRunningQuery extends LogMetricId("slow-running-query")
+ case NonIndexedQuery extends LogMetricId("non-indexed-query" )
+ case UnsafeContent extends LogMetricId("unsafe-content" )
+
+ case class LogMetric(
+ id : LogMetricId
+ , displayName: String
+ , logType : LogConfigType
+ , kibanaLink : String
+ )
+
+ enum LogConfigType(val query: String):
+ case GenericSearch( override val query: String) extends LogConfigType(query)
+ case AverageMongoDuration(override val query: String) extends LogConfigType(query)
+
+
+
+

app/uk/gov/hmrc/servicemetrics/connector/ClickHouseConnector.scala

@@ -16,26 +16,34 @@
 
 package uk.gov.hmrc.servicemetrics.connector
 
+import play.api.Configuration
 import play.api.libs.json.JsValue
 import uk.gov.hmrc.http.HttpReads.Implicits._
 import uk.gov.hmrc.http.{HeaderCarrier, StringContextOps}
 import uk.gov.hmrc.http.client.HttpClientV2
-import uk.gov.hmrc.servicemetrics.config.ClickHouseConfig
 import uk.gov.hmrc.servicemetrics.model.Environment
 
 import javax.inject.{Inject, Singleton}
 import scala.concurrent.{ExecutionContext, Future}
 
 @Singleton
 class ClickHouseConnector @Inject()(
- httpClientV2: HttpClientV2
-, config : ClickHouseConfig
+ configuration: Configuration
+, httpClientV2 : HttpClientV2
 )(using
  ExecutionContext
 ):
 
+ private val environmentUrls: Map[Environment, String] =
+ Environment
+ .values
+ .filterNot(_ == Environment.Integration)
+ .map:
+ env => env -> configuration.get[String](s"clickhouse.${env.asString}.url")
+ .toMap
+
  def getDatabaseNames(environment: Environment)(using HeaderCarrier): Future[Seq[String]] =
  httpClientV2
- .get(url"${config.urls(environment)}/latest/mongodbs")
+ .get(url"${environmentUrls(environment)}/latest/mongodbs")
  .execute[JsValue]
  .map(json => (json \ "name").as[Seq[String]])