Sbt project modernization (close #361)

.github/workflows/deploy.yml

@@ -23,7 +23,6 @@ jobs:
           sbt 'project pubsub' assembly
           sbt 'project sqs' assembly
           sbt 'project stdout' assembly
-          sbt 'project rabbitmq' assembly
       - name: Get current version
         id: ver
         run: |
@@ -43,7 +42,6 @@ jobs:
             pubsub/target/scala-2.12/snowplow-stream-collector-google-pubsub-${{ steps.ver.outputs.project_version }}.jar
             sqs/target/scala-2.12/snowplow-stream-collector-sqs-${{ steps.ver.outputs.project_version }}.jar
             stdout/target/scala-2.12/snowplow-stream-collector-stdout-${{ steps.ver.outputs.project_version }}.jar
-            rabbitmq/target/scala-2.12/snowplow-stream-collector-stdout-${{ steps.ver.outputs.project_version }}.jar
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -58,11 +56,8 @@ jobs:
           - kafka
           - nsq
           - stdout
-          - rabbitmq
         include:
           - suffix: ""
-          - suffix: -experimental
-            platform: rabbitmq
           - platform: kinesis
             run_snyk: ${{ !contains(github.ref, 'rc') }}
           - platform: pubsub

build.sbt

@@ -6,128 +6,18 @@
  * and you may not use this file except in compliance with the Snowplow Community License Version 1.0.
  * You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0
  */
-import com.typesafe.sbt.packager.docker._
-import sbtbuildinfo.BuildInfoPlugin.autoImport._
-
-lazy val commonDependencies = Seq(
-  // Java
-  Dependencies.Libraries.thrift,
-  Dependencies.Libraries.jodaTime,
-  Dependencies.Libraries.slf4j,
-  Dependencies.Libraries.log4jOverSlf4j,
-  Dependencies.Libraries.config,
-  // Scala
-  Dependencies.Libraries.scopt,
-  Dependencies.Libraries.akkaStream,
-  Dependencies.Libraries.akkaHttp,
-  Dependencies.Libraries.akkaStream,
-  Dependencies.Libraries.akkaSlf4j,
-  Dependencies.Libraries.akkaHttpMetrics,
-  Dependencies.Libraries.jnrUnixsocket,
-  Dependencies.Libraries.badRows,
-  Dependencies.Libraries.collectorPayload,
-  Dependencies.Libraries.pureconfig,
-  Dependencies.Libraries.Legacy.trackerCore,
-  Dependencies.Libraries.Legacy.trackerEmitterId,
-  // Unit tests
-  Dependencies.Libraries.akkaTestkit,
-  Dependencies.Libraries.akkaHttpTestkit,
-  Dependencies.Libraries.akkaStreamTestkit,
-  Dependencies.Libraries.specs2,
-  // Integration tests
-  Dependencies.Libraries.Legacy.testcontainers,
-  Dependencies.Libraries.Legacy.http4sClient,
-  Dependencies.Libraries.Legacy.catsRetry
-)
-
-lazy val commonExclusions = Seq(
-  "org.apache.tomcat.embed" % "tomcat-embed-core", // exclude for security vulnerabilities introduced by libthrift
-  // Avoid duplicate .proto files brought in by akka and google-cloud-pubsub.
-  // We don't need any akka serializers because collector runs in a single JVM.
-  "com.typesafe.akka" % "akka-protobuf-v3_2.12"
-)
-
-lazy val buildInfoSettings = Seq(
-  buildInfoPackage := "com.snowplowanalytics.snowplow.collectors.scalastream.generated",
-  buildInfoKeys := Seq[BuildInfoKey](organization, moduleName, name, version, "shortName" -> "ssc", scalaVersion)
-)
-
-// Make package (build) metadata available within source code for integration tests.
-lazy val scalifiedSettings = Seq(
-  IntegrationTest / sourceGenerators += Def.task {
-    val file = (IntegrationTest / sourceManaged).value / "settings.scala"
-    IO.write(
-      file,
-      """package %s
-        |object ProjectMetadata {
-        |  val organization = "%s"
-        |  val name = "%s"
-        |  val version = "%s"
-        |  val dockerTag = "%s"
-        |}
-        |"""
-        .stripMargin
-        .format(
-          buildInfoPackage.value,
-          organization.value,
-          name.value,
-          version.value,
-          dockerAlias.value.tag.get
-        )
-    )
-    Seq(file)
-  }.taskValue
-)
-
-lazy val buildSettings = Seq(
-  organization := "com.snowplowanalytics",
-  name := "snowplow-stream-collector",
-  description := "Scala Stream Collector for Snowplow raw events",
-  scalaVersion := "2.12.10",
-  scalacOptions ++= Seq("-Ypartial-unification", "-Ywarn-macros:after"),
-  javacOptions := Seq("-source", "11", "-target", "11"),
-  resolvers ++= Dependencies.resolutionRepos
-)
-
-lazy val dynVerSettings = Seq(
-  ThisBuild / dynverVTagPrefix := false, // Otherwise git tags required to have v-prefix
-  ThisBuild / dynverSeparator := "-"     // to be compatible with docker
-)
-
-lazy val http4sBuildInfoSettings = Seq(
-  buildInfoKeys := Seq[BuildInfoKey](name, moduleName, dockerAlias, version),
-  buildInfoOptions += BuildInfoOption.Traits("com.snowplowanalytics.snowplow.collector.core.AppInfo")
-)
-
-lazy val allSettings = buildSettings ++
-  BuildSettings.sbtAssemblySettings ++
-  BuildSettings.formatting ++
-  Seq(excludeDependencies ++= commonExclusions) ++
-  dynVerSettings ++
-  BuildSettings.addExampleConfToTestCp
-
+
 lazy val root = project
   .in(file("."))
-  .settings(buildSettings ++ dynVerSettings)
-  .aggregate(core, kinesis, pubsub, kafka, nsq, stdout, sqs, rabbitmq, http4s)
-
-lazy val core = project
-  .settings(moduleName := "snowplow-stream-collector-core")
-  .settings(buildSettings ++ BuildSettings.sbtAssemblySettings)
-  .settings(libraryDependencies ++= commonDependencies)
-  .settings(excludeDependencies ++= commonExclusions)
-  .settings(Defaults.itSettings)
-  .configs(IntegrationTest)
+  .aggregate(kinesis, pubsub, kafka, nsq, stdout, sqs, http4s)
 
 lazy val http4s = project
   .settings(moduleName := "snowplow-stream-collector-http4s-core")
-  .settings(buildSettings ++ BuildSettings.sbtAssemblySettings)
+  .settings(BuildSettings.coreHttp4sSettings)
   .settings(
     libraryDependencies ++= Seq(
       Dependencies.Libraries.http4sDsl,
-      Dependencies.Libraries.http4sEmber,
       Dependencies.Libraries.http4sBlaze,
-      Dependencies.Libraries.http4sNetty,
       Dependencies.Libraries.http4sClient,
       Dependencies.Libraries.log4cats,
       Dependencies.Libraries.thrift,
@@ -144,192 +34,88 @@ lazy val http4s = project
       Dependencies.Libraries.ceTestkit,
 
       //Integration tests
-      Dependencies.Libraries.IT.testcontainers,
-      Dependencies.Libraries.IT.http4sClient,
-      Dependencies.Libraries.IT.catsRetry
+      Dependencies.Libraries.IntegrationTests.testcontainers,
+      Dependencies.Libraries.IntegrationTests.http4sClient,
+      Dependencies.Libraries.IntegrationTests.catsRetry
 
     )
   )
-  .settings(Defaults.itSettings)
   .configs(IntegrationTest)
 
-lazy val kinesisSettings =
-  allSettings ++ buildInfoSettings ++ http4sBuildInfoSettings ++ Defaults.itSettings ++ scalifiedSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-kinesis",
-    buildInfoPackage := s"com.snowplowanalytics.snowplow.collectors.scalastream",
-    Docker / packageName := "scala-stream-collector-kinesis",
-    libraryDependencies ++= Seq(
-      Dependencies.Libraries.catsRetry,
-      Dependencies.Libraries.kinesis,
-      Dependencies.Libraries.sts,
-      Dependencies.Libraries.sqs,
-      // integration tests dependencies
-      Dependencies.Libraries.IT.specs2,
-      Dependencies.Libraries.IT.specs2CE,
-    ),
-    IntegrationTest / test := (IntegrationTest / test).dependsOn(Docker / publishLocal).value,
-    IntegrationTest / testOnly := (IntegrationTest / testOnly).dependsOn(Docker / publishLocal).evaluated
-  )
-
 lazy val kinesis = project
-  .settings(kinesisSettings)
+  .settings(BuildSettings.kinesisSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile;it->it")
   .configs(IntegrationTest)
 
 lazy val kinesisDistroless = project
   .in(file("distroless/kinesis"))
   .settings(sourceDirectory := (kinesis / sourceDirectory).value)
-  .settings(kinesisSettings)
+  .settings(BuildSettings.kinesisSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile;it->it")
   .configs(IntegrationTest)
 
-lazy val sqsSettings =
-  allSettings ++ buildInfoSettings ++ http4sBuildInfoSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-sqs",
-    buildInfoPackage := s"com.snowplowanalytics.snowplow.collectors.scalastream",
-    Docker / packageName := "scala-stream-collector-sqs",
-    libraryDependencies ++= Seq(
-      Dependencies.Libraries.catsRetry,
-      Dependencies.Libraries.sqs,
-      Dependencies.Libraries.sts,
-    )
-  )
-
 lazy val sqs = project
-  .settings(sqsSettings)
+  .settings(BuildSettings.sqsSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile")
 
 lazy val sqsDistroless = project
   .in(file("distroless/sqs"))
   .settings(sourceDirectory := (sqs / sourceDirectory).value)
-  .settings(sqsSettings)
+  .settings(BuildSettings.sqsSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile")
 
-lazy val pubsubSettings =
-  allSettings ++ buildInfoSettings ++ http4sBuildInfoSettings ++ Defaults.itSettings ++ scalifiedSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-google-pubsub",
-    buildInfoPackage := s"com.snowplowanalytics.snowplow.collectors.scalastream",
-    Docker / packageName := "scala-stream-collector-pubsub",
-    libraryDependencies ++= Seq(
-      Dependencies.Libraries.catsRetry,
-      Dependencies.Libraries.fs2PubSub,
-      // integration tests dependencies
-      Dependencies.Libraries.IT.specs2, 
-      Dependencies.Libraries.IT.specs2CE,
-    ),
-    IntegrationTest / test := (IntegrationTest / test).dependsOn(Docker / publishLocal).value,
-    IntegrationTest / testOnly := (IntegrationTest / testOnly).dependsOn(Docker / publishLocal).evaluated
-  )
-
 lazy val pubsub = project
-  .settings(pubsubSettings)
+  .settings(BuildSettings.pubsubSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile;it->it")
   .configs(IntegrationTest)
 
 lazy val pubsubDistroless = project
   .in(file("distroless/pubsub"))
   .settings(sourceDirectory := (pubsub / sourceDirectory).value)
-  .settings(pubsubSettings)
+  .settings(BuildSettings.pubsubSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile;it->it")
   .configs(IntegrationTest)
 
-lazy val kafkaSettings =
-  allSettings ++ buildInfoSettings ++ http4sBuildInfoSettings ++ Defaults.itSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-kafka",
-    buildInfoPackage := s"com.snowplowanalytics.snowplow.collectors.scalastream",
-    Docker / packageName := "scala-stream-collector-kafka",
-    libraryDependencies ++= Seq(
-      Dependencies.Libraries.kafkaClients,
-      Dependencies.Libraries.mskAuth,
-      // integration tests dependencies
-      Dependencies.Libraries.IT.specs2,
-      Dependencies.Libraries.IT.specs2CE
-    ),
-    IntegrationTest / test := (IntegrationTest / test).dependsOn(Docker / publishLocal).value,
-    IntegrationTest / testOnly := (IntegrationTest / testOnly).dependsOn(Docker / publishLocal).evaluated
-  )
-
 lazy val kafka = project
-  .settings(kafkaSettings)
+  .settings(BuildSettings.kafkaSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile;it->it")
   .configs(IntegrationTest)
 
-
 lazy val kafkaDistroless = project
   .in(file("distroless/kafka"))
   .settings(sourceDirectory := (kafka / sourceDirectory).value)
-  .settings(kafkaSettings)
+  .settings(BuildSettings.kafkaSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile;it->it")
   .configs(IntegrationTest)
 
-
-lazy val nsqSettings =
-  allSettings ++ buildInfoSettings ++ http4sBuildInfoSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-nsq",
-    Docker / packageName := "scala-stream-collector-nsq",
-    buildInfoPackage := s"com.snowplowanalytics.snowplow.collectors.scalastream",
-    libraryDependencies ++= Seq(
-      Dependencies.Libraries.nsqClient,
-      Dependencies.Libraries.jackson,
-      Dependencies.Libraries.nettyAll,
-      Dependencies.Libraries.log4j
-    )
-  )
-
 lazy val nsq = project
-  .settings(nsqSettings)
+  .settings(BuildSettings.nsqSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile")
 
 lazy val nsqDistroless = project
   .in(file("distroless/nsq"))
   .settings(sourceDirectory := (nsq / sourceDirectory).value)
-  .settings(nsqSettings)
+  .settings(BuildSettings.nsqSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile")
 
-lazy val stdoutSettings =
-  allSettings ++ buildInfoSettings ++ http4sBuildInfoSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-stdout",
-    buildInfoPackage := s"com.snowplowanalytics.snowplow.collector.stdout",
-    Docker / packageName := "scala-stream-collector-stdout"
-  )
-
 lazy val stdout = project
-  .settings(stdoutSettings)
+  .settings(BuildSettings.stdoutSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
   .dependsOn(http4s % "test->test;compile->compile")
 
 lazy val stdoutDistroless = project
   .in(file("distroless/stdout"))
   .settings(sourceDirectory := (stdout / sourceDirectory).value)
-  .settings(stdoutSettings)
-  .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
-  .dependsOn(http4s % "test->test;compile->compile")
-
-lazy val rabbitmqSettings =
-  allSettings ++ buildInfoSettings ++ Seq(
-    moduleName := "snowplow-stream-collector-rabbitmq",
-    Docker / packageName := "scala-stream-collector-rabbitmq-experimental",
-    libraryDependencies ++= Seq(Dependencies.Libraries.rabbitMQ)
-  )
-
-lazy val rabbitmq = project
-  .settings(rabbitmqSettings)
-  .enablePlugins(JavaAppPackaging, SnowplowDockerPlugin, BuildInfoPlugin)
-  .dependsOn(core % "test->test;compile->compile")
-
-lazy val rabbitmqDistroless = project
-  .in(file("distroless/rabbitmq"))
-  .settings(sourceDirectory := (rabbitmq / sourceDirectory).value)
-  .settings(rabbitmqSettings)
+  .settings(BuildSettings.stdoutSettings)
   .enablePlugins(JavaAppPackaging, SnowplowDistrolessDockerPlugin, BuildInfoPlugin)
-  .dependsOn(core % "test->test;compile->compile")
+  .dependsOn(http4s % "test->test;compile->compile")