diff --git a/build.sbt b/build.sbt deleted file mode 100644 index e862922743..0000000000 --- a/build.sbt +++ /dev/null @@ -1,627 +0,0 @@ -import Dependencies._ -import BuildSettings._ -import sbtassembly.AssemblyPlugin.autoImport.assemblyMergeStrategy -import com.typesafe.sbt.packager.docker._ - -lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, embeddedPg, embeddedPgSupport, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi) - -lazy val sparkDeps = - Seq( - postgres, - akkaKafkaStreams - ) - -lazy val authDeps = Seq(keycloakAdapter, keycloak, jbossLogging, httpClient) - -lazy val akkaDeps = Seq( - akkaSlf4J, - akkaCluster, - akkaTyped, - akkaClusterTyped, - akkaStream, - akkaStreamTyped, - akkaManagement, - akkaManagementClusterBootstrap, - akkaServiceDiscoveryDNS, - akkaServiceDiscoveryKubernetes, - akkaClusterHttpManagement, - akkaClusterHttpManagement, - akkaTestkitTyped, - akkaStreamsTestKit, - akkaCors, - akkaKafkaStreams, - embeddedKafka, - alpakkaS3, - akkaQuartzScheduler, - alpakkaFile -) - -lazy val akkaPersistenceDeps = - Seq( - akkaPersistence, - akkaClusterSharding, - akkaPersistenceTyped, - akkaPersistenceQuery, - akkaClusterShardingTyped, - akkaPersistenceCassandra, - cassandraLauncher - ) - -lazy val akkaHttpDeps = - Seq(akkaHttp, akkaHttp2, akkaHttpTestkit, akkaStreamsTestKit, akkaHttpCirce, akkaHttpXml) -lazy val circeDeps = Seq(circe, circeGeneric, circeParser) -lazy val enumeratumDeps = Seq(enumeratum, enumeratumCirce) - -lazy val slickDeps = Seq(slick, slickHikariCP, postgres, h2) - -lazy val dockerSettings = Seq( - Docker / maintainer := "Hmda-Ops", - dockerBaseImage := "eclipse-temurin:19-jdk-alpine", - dockerRepository := Some("hmda"), - dockerCommands := dockerCommands.value.flatMap { - case cmd@Cmd("FROM",_) => List(cmd, Cmd("RUN", "apk update"), - Cmd("RUN", "rm /var/cache/apk/*")) - case other => List(other) - } -) - -lazy val packageSettings = Seq( - // removes all jar mappings in universal and appends the fat jar - mappings in Universal := { - // universalMappings: Seq[(File,String)] - val universalMappings = (mappings in Universal).value - val fatJar = (assembly in Compile).value - // removing means filtering - val filtered = universalMappings filter { - case (_, fileName) => !fileName.endsWith(".jar") || fileName.contains("cinnamon-agent") - } - // add the fat jar - filtered :+ (fatJar -> ("lib/" + fatJar.getName)) - }, - // the bash scripts classpath only needs the fat jar - scriptClasspath := Seq((assemblyJarName in assembly).value) -) - -lazy val `hmda-root` = (project in file(".")) - .settings(hmdaBuildSettings: _*) - .aggregate( - common, - `hmda-platform`, - `check-digit`, - `file-proxy`, - `institutions-api`, - `modified-lar`, - `hmda-analytics`, - `hmda-data-publisher`, - `hmda-reporting`, - `ratespread-calculator`, - `data-browser`, - `submission-errors`, - `hmda-quarterly-data-service` - ) - -val latestGitTag = settingKey[String]("The latest git tag.") -ThisBuild / latestGitTag := { - import scala.sys.process._ - - val hasTags = "git tag".lineStream_!.nonEmpty - - if (hasTags) { - "git describe --tags".lineStream_!.head - } else { - Keys.sLog.value.warn("No git tags in the checkout, using '-' for build info.") - "-" - } -} - -lazy val common = (project in file("common")) - .settings(hmdaBuildSettings: _*) - .settings( - PB.targets in Compile := Seq( - scalapb.gen() -> (sourceManaged in Compile).value / "protobuf" - ), - Seq( - libraryDependencies ++= commonDeps ++ authDeps ++ akkaDeps ++ akkaPersistenceDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ List( - cormorant, cormorantGeneric, scalaMock, scalacheckShapeless, diffx - ) - ), - addCompilerPlugin("com.olegpy" %% "better-monadic-for" % "0.3.1") - ) - .enablePlugins(BuildInfoPlugin) - .settings( - buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, latestGitTag), - buildInfoPackage := "hmda" - ) - -lazy val `hmda-platform` = (project in file("hmda")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies += zeroAllocationHashing, - mainClass in Compile := Some("hmda.HmdaPlatform"), - assemblyJarName in assembly := "hmda2.jar", - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "cinnamon-reference.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case "logback.xml" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - envVars in reStart ++= Map("CASSANDRA_CLUSTER_HOSTS" -> "localhost", "APP_PORT" -> "2551"), - addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full) - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol` % "compile->compile;test->test") - -lazy val `check-digit` = (project in file("check-digit")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.uli.HmdaUli"), - assemblyJarName in assembly := { - s"${name.value}.jar" - }, - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol` % "compile->compile;test->test") - - lazy val `file-proxy` = (project in file("file-proxy")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ - enumeratumDeps :+ monix :+ lettuce :+ scalaJava8Compat :+ scalaMock, - mainClass in Compile := Some("hmda.proxy.FileProxy"), - assemblyJarName in assembly := { - s"${name.value}.jar" - }, - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - - -lazy val `institutions-api` = (project in file("institutions-api")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.institution.HmdaInstitutionApi"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - -lazy val `hmda-data-publisher` = (project in file("hmda-data-publisher")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ enumeratumDeps :+ - scalaMock :+ cormorantGeneric :+ scalacheckShapeless :+ diffx, - mainClass in Compile := Some("hmda.publisher.HmdaDataPublisherApp"), - assemblyJarName in assembly := { - s"${name.value}.jar" - }, - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - } - ), - dockerSettings, - packageSettings, - addCompilerPlugin("com.olegpy" %% "better-monadic-for" % "0.3.1") - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol` % "compile->compile;test->test") - -lazy val `hmda-dashboard` = (project in file("hmda-dashboard")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ - enumeratumDeps :+ monix :+ lettuce :+ scalaJava8Compat :+ scalaMock, - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol` % "compile->compile;test->test") - -lazy val `ratespread-calculator` = (project in file("ratespread-calculator")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.calculator.HmdaRateSpread"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol` % "compile->compile;test->test") - -lazy val `modified-lar` = (project in file("modified-lar")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.publication.lar.ModifiedLarApp"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol`) - .dependsOn(common) - -lazy val `irs-publisher` = (project in file("irs-publisher")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.publication.lar.IrsPublisherApp"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol`) - .dependsOn(common) - -lazy val `hmda-reporting` = (project in file("hmda-reporting")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.reporting.HmdaReporting"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol`) - .dependsOn(common) - -lazy val `hmda-protocol` = (project in file("protocol")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin, - AkkaGrpcPlugin - ) - .settings(hmdaBuildSettings: _*) - -lazy val `hmda-analytics` = (project in file("hmda-analytics")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.analytics.HmdaAnalyticsApp"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - -lazy val `rate-limit` = (project in file("rate-limit")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps :+ guava, - mainClass in Compile := Some("hmda.rateLimit.RateLimitApp"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol`) - -lazy val `data-browser` = (project in file("data-browser")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ - enumeratumDeps :+ monix :+ lettuce :+ scalaJava8Compat :+ scalaMock, - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - -lazy val `submission-errors` = (project in file("submission-errors")) - .enablePlugins(JavaServerAppPackaging, sbtdocker.DockerPlugin, AshScriptPlugin) - .settings(hmdaBuildSettings) - .settings( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps :+ monix :+ slickPostgres, - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") // allows you to depend on code from both compile and test scopes - -lazy val `email-service` = (project in file("email-service")) - .enablePlugins(JavaServerAppPackaging, sbtdocker.DockerPlugin, AshScriptPlugin) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - mainClass in Compile := Some("hmda.publication.lar.EmailReceiptApp"), - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - }, - libraryDependencies ++= monix :: akkaKafkaStreams :: awsSesSdk :: logback :: Nil - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol`) - -lazy val `hmda-quarterly-data-service` = (project in file ("hmda-quarterly-data-service")) - .enablePlugins( - JavaServerAppPackaging, - sbtdocker.DockerPlugin, - AshScriptPlugin - ) - .settings(hmdaBuildSettings: _*) - .settings( - Seq( - libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ - enumeratumDeps :+ monix :+ lettuce :+ scalaJava8Compat :+ scalaMock, - assemblyMergeStrategy in assembly := { - case "application.conf" => MergeStrategy.concat - case "META-INF/io.netty.versions.properties" => MergeStrategy.concat - case PathList(ps @ _*) if ps.last endsWith ".proto" => - MergeStrategy.first - case "module-info.class" => MergeStrategy.concat - case x if x.endsWith("/module-info.class") => MergeStrategy.concat - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) - }, - assemblyJarName in assembly := { - s"${name.value}.jar" - } - ), - dockerSettings, - packageSettings - ) - .dependsOn(common % "compile->compile;test->test") - .dependsOn(`hmda-protocol` % "compile->compile;test->test") diff --git a/common/src/main/resources/persistence.conf b/common/src/main/resources/persistence.conf deleted file mode 100644 index 8dac1c6e70..0000000000 --- a/common/src/main/resources/persistence.conf +++ /dev/null @@ -1,88 +0,0 @@ -include "serialization.conf" - -akka { - - persistence { - journal.plugin = "akka.persistence.cassandra.journal" - snapshot-store.plugin = "akka.persistence.cassandra.snapshot" - - query { - journal.id = "akka.persistence.cassandra.query" - } - - cassandra { - events-by-tag { - max-message-batch-size = 30 - max-message-batch-size = ${?TAG_BATCH_SIZE} - flush-interval = 150ms - flush-interval = ${?TAG_FLUSH_INTERVAL} - } - journal { - keyspace = "hmda2_journal" - keyspace = ${?CASSANDRA_JOURNAL_KEYSPACE} - table = "journal" - events-by-tag.max-message-batch-size = 30 - events-by-tag.max-message-batch-size = ${?TAG_BATCH_SIZE} - events-by-tag.flush-interval = 150ms - events-by-tag.flush-interval = ${?TAG_FLUSH_INTERVAL} - } - query { - # reference: https://github.com/akka/akka-persistence-cassandra/blob/v0.101/core/src/main/resources/reference.conf - # Sequence numbers for a persistenceId is assumed to be monotonically increasing - # without gaps. That is used for detecting missing events. - # In early versions of the journal that might not be true and therefore - # this can be relaxed by setting this property to off. - gap-free-sequence-numbers = off - } - snapshot { - keyspace = "hmda2_snapshot" - keyspace = ${?CASSANDRA_SNAPSHOT_KEYSPACE} - table = "snapshot" - } - } - } - - -} - -datastax-java-driver { - basic { - contact-points = ["localhost:9042"] - contact-points = [${?CASSANDRA_CLUSTER_HOSTS}":9042"] - load-balancing-policy.local-datacenter = "" - load-balancing-policy.local-datacenter = ${?CASSANDRA_CLUSTER_DC} - } - advanced { - auth-provider { - class = PlainTextAuthProvider - username = "" - username = ${?CASSANDRA_CLUSTER_USERNAME} - password = "" - password = ${?CASSANDRA_CLUSTER_PASSWORD} - } - request-tracker { - class = RequestLogger - logs { - slow { - threshold = 1 second - enabled = true - } - } - } - } -} - -kafka { - hosts = "localhost:9092" - hosts = ${?KAFKA_CLUSTER_HOSTS} - idle-timeout = 5 - idle-timeout = ${?KAFKA_IDLE_TIMEOUT} - security.protocol="" - security.protocol=${?KAFKA_SECURITY} - ssl.truststore.location = "" - ssl.truststore.location = ${?TRUSTSTORE_PATH} - ssl.truststore.password = "" - ssl.truststore.password = ${?TRUSTSTORE_PASSWORD} - ssl.endpoint = "" - ssl.endpoint = ${?KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG} -} diff --git a/kubernetes/config-maps/cassandra-configmap.yaml b/kubernetes/config-maps/cassandra-configmap.yaml index 4ac0d41abd..df1cbcc612 100644 --- a/kubernetes/config-maps/cassandra-configmap.yaml +++ b/kubernetes/config-maps/cassandra-configmap.yaml @@ -2,10 +2,9 @@ apiVersion: v1 kind: ConfigMap metadata: name: cassandra-configmap - annotations: - reloader.stakater.com/match: "true" + namespace: default data: - cassandra-hosts: cluster1-dc1-service.cass-operator - cassandra-cluster-dc: dc1 + cassandra-cluster-dc: us-east-1 + cassandra-hosts-port: cassandra.us-east-1.amazonaws.com:9142 cassandra-keyspace: hmda2_journal cassandra-keyspace-snapshot: hmda2_snapshot diff --git a/kubernetes/hmda-platform/README.md b/kubernetes/hmda-platform/README.md deleted file mode 100644 index 6aa5695df4..0000000000 --- a/kubernetes/hmda-platform/README.md +++ /dev/null @@ -1,37 +0,0 @@ -Pre-requisites -- [k8ssandra](https://k8ssandra.io/) -- [Strimzi](https://strimzi.io/) -- Create schema for platform -``` -cqlsh -u -``` -Install -- Add Secrets -``` -kubectl create secret generic cassandra-credentials --from-literal=cassandra.username= --from-literal=cassandra.password= -kubectl create secret generic inst-postgres-credentials --from-literal=username= --from-literal=password= --from-literal=host= --from-literal=url="jdbc:postgresql://postgresql:5432/hmda?user= &password= &sslmode=false" -``` -- Add Configmap -``` -kubectl apply -f https://github.com/cfpb/hmda-platform/tree/master/kubernetes/config-maps -``` -- Create schema for platform -``` -cqlsh -u username -p passsword -f ../hmda-sql-doc/cassandra-scripts.txt -``` - -Update -``` -helm upgrade --install --namespace=default --values=kubernetes/hmda-platform/values-dev.yaml \ ---set image.tag=latest \ ---set service.name=hmda-platform \ -hmda-platform kubernetes/hmda-platform -``` - -Delete -``` -helm uninstall hmda-platform -kubectl delete secret cassandra-credentials -kubectl delete secret inst-postgres-credentials -kubectl delete -f https://github.com/cfpb/hmda-platform/tree/master/kubernetes/config-maps -``` diff --git a/kubernetes/hmda-platform/templates/deployment.yaml b/kubernetes/hmda-platform/templates/deployment.yaml index 0d96668608..e0d1242464 100644 --- a/kubernetes/hmda-platform/templates/deployment.yaml +++ b/kubernetes/hmda-platform/templates/deployment.yaml @@ -30,6 +30,9 @@ spec: - name: tz-config hostPath: path: /usr/share/zoneinfo/America/New_York + - name: cassandra-truststore + secret: + secretName: cassandra-truststore serviceAccountName: {{ .Values.service.account.name }} containers: - name: {{ .Chart.Name }} @@ -37,6 +40,9 @@ spec: volumeMounts: - name: tz-config mountPath: /etc/localtime + - name: cassandra-truststore + readOnly: true + mountPath: "/opt/cassandra_truststore" resources: {{ toYaml .Values.resources | indent 12 }} securityContext: @@ -149,11 +155,13 @@ spec: fieldRef: apiVersion: v1 fieldPath: status.podIP - - name: CASSANDRA_CLUSTER_HOSTS + - name: CASSANDRA_TRUSTSTORE_FILE + value: "/opt/cassandra_truststore/cassandra_truststore.jks" + - name: CASSANDRA_CLUSTER_HOSTS_PORT valueFrom: configMapKeyRef: + key: cassandra-hosts-port name: cassandra-configmap - key: cassandra-hosts - name: CASSANDRA_CLUSTER_DC valueFrom: configMapKeyRef: @@ -174,11 +182,26 @@ spec: secretKeyRef: name: cassandra-credentials key: cassandra.username + - name: CASSANDRA_TRUSTSTORE_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: cassandra-truststore-password - name: CASSANDRA_CLUSTER_PASSWORD valueFrom: secretKeyRef: name: cassandra-credentials key: cassandra.password + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: aws-access-key-id + name: cassandra-keyspace-credentials + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: aws-secret-access-key + name: cassandra-keyspace-credentials - name: CASSANDRA_LOG_LEVEL value: {{ .Values.logs.cassandra }} - name: KEYCLOAK_REALM_URL diff --git a/project/Dependencies.scala b/project/Dependencies.scala deleted file mode 100644 index 2573985a87..0000000000 --- a/project/Dependencies.scala +++ /dev/null @@ -1,89 +0,0 @@ -import sbt._ - -object Dependencies { - - val repos = Seq( - "Local Maven Repo" at "file://" + Path.userHome.absolutePath + "/.m2/repository", - "Typesafe Repo" at "http://repo.typesafe.com/typesafe/releases/", - "Sonatype releases" at "https://oss.sonatype.org/content/repositories/releases" - ) - - val scalaTest = "org.scalatest" %% "scalatest" % Version.scalaTest % Test - val scalaCheck = "org.scalacheck" %% "scalacheck" % Version.scalaCheck % Test - val scalaMock = "org.scalamock" %% "scalamock" % Version.scalaMock % Test - val logback = "ch.qos.logback" % "logback-classic" % Version.logback - lazy val akkaSlf4J = "com.typesafe.akka" %% "akka-slf4j" % Version.akka - lazy val akkaTyped = "com.typesafe.akka" %% "akka-actor-typed" % Version.akka - lazy val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % Version.akka - lazy val akkaClusterTyped = "com.typesafe.akka" %% "akka-cluster-typed" % Version.akka - lazy val akkaClusterSharding = "com.typesafe.akka" %% "akka-cluster-sharding" % Version.akka - lazy val akkaClusterShardingTyped = "com.typesafe.akka" %% "akka-cluster-sharding-typed" % Version.akka - lazy val akkaPersistence = "com.typesafe.akka" %% "akka-persistence" % Version.akka - lazy val akkaPersistenceTyped = "com.typesafe.akka" %% "akka-persistence-typed" % Version.akka - lazy val akkaPersistenceQuery = "com.typesafe.akka" %% "akka-persistence-query" % Version.akka - lazy val akkaTestkitTyped = "com.typesafe.akka" %% "akka-actor-testkit-typed" % Version.akka % Test - lazy val akkaStream = "com.typesafe.akka" %% "akka-stream" % Version.akka - lazy val akkaStreamTyped = "com.typesafe.akka" %% "akka-stream-typed" % Version.akka - lazy val akkaStreamsTestKit = "com.typesafe.akka" %% "akka-stream-testkit" % Version.akka % Test - lazy val akkaHttp = "com.typesafe.akka" %% "akka-http" % Version.akkaHttp - lazy val akkaHttp2 = "com.typesafe.akka" %% "akka-http2-support" % Version.akkaHttp - lazy val akkaHttpTestkit = "com.typesafe.akka" %% "akka-http-testkit" % Version.akkaHttp % Test - lazy val akkaHttpSprayJson = "com.typesafe.akka" %% "akka-http-spray-json" % Version.akkaHttp - lazy val slickPostgres = "com.github.tminglei" %% "slick-pg" % Version.slickPostgres - lazy val akkaHttpXml = "com.typesafe.akka" %% "akka-http-xml" % Version.akkaHttp - lazy val akkaHttpCirce = "de.heikoseeberger" %% "akka-http-circe" % Version.akkaHttpJson - lazy val akkaManagementClusterBootstrap = - "com.lightbend.akka.management" %% "akka-management-cluster-bootstrap" % Version.akkaClusterManagement exclude ("com.typesafe.akka", "akka-http") exclude ("com.typesafe.akka", "akka-http-xml") - lazy val akkaServiceDiscoveryDNS = - "com.typesafe.akka" %% "akka-discovery" % Version.akka exclude ("com.typesafe.akka", "akka-http") exclude ("com.typesafe.akka", "akka-http-xml") - lazy val akkaServiceDiscoveryKubernetes = - "com.lightbend.akka.discovery" %% "akka-discovery-kubernetes-api" % Version.akkaClusterManagement exclude ("com.typesafe.akka", "akka-http") exclude ("com.typesafe.akka", "akka-http-xml") - lazy val akkaManagement = - "com.lightbend.akka.management" %% "akka-management" % Version.akkaClusterManagement exclude ("com.typesafe.akka", "akka-http") exclude ("com.typesafe.akka", "akka-http-xml") - lazy val akkaClusterHttpManagement = - "com.lightbend.akka.management" %% "akka-management-cluster-http" % Version.akkaClusterManagement exclude ("com.typesafe.akka", "akka-http") exclude ("com.typesafe.akka", "akka-http-xml") - lazy val akkaCors = - "ch.megard" %% "akka-http-cors" % Version.akkaCors exclude ("com.typesafe.akka", "akka-http") exclude ("com.typesafe.akka", "akka-http-xml") - lazy val circe = "io.circe" %% "circe-core" % Version.circe - lazy val circeGeneric = "io.circe" %% "circe-generic" % Version.circe - lazy val circeParser = "io.circe" %% "circe-parser" % Version.circe - lazy val akkaPersistenceCassandra = "com.typesafe.akka" %% "akka-persistence-cassandra" % Version.cassandraPluginVersion - lazy val cassandraLauncher = "com.typesafe.akka" %% "akka-persistence-cassandra-launcher" % Version.cassandraPluginVersion - lazy val slick = "com.typesafe.slick" %% "slick" % Version.slick - lazy val slickHikariCP = "com.typesafe.slick" %% "slick-hikaricp" % Version.slick - lazy val alpakkaSlick = "com.lightbend.akka" %% "akka-stream-alpakka-slick" % Version.alpakkaSlick - lazy val postgres = "org.postgresql" % "postgresql" % Version.postgres - lazy val h2 = "com.h2database" % "h2" % Version.h2 % Test - lazy val embeddedPg = - "ru.yandex.qatools.embed" % "postgresql-embedded" % Version.embeddedPg % Test exclude ("de.flapdoodle.embed", "de.flapdoodle.embed.process") - lazy val embeddedPgSupport = "de.flapdoodle.embed" % "de.flapdoodle.embed.process" % "2.1.2" % Test - lazy val s3Mock = "com.adobe.testing" % "s3mock" % "2.1.19" % Test - lazy val apacheCommonsIO = "commons-io" % "commons-io" % "2.6" % Test - lazy val keycloakAdapter = "org.keycloak" % "keycloak-adapter-core" % Version.keycloak - lazy val keycloak = "org.keycloak" % "keycloak-core" % Version.keycloak - lazy val jbossLogging = "org.jboss.logging" % "jboss-logging" % Version.jboss - lazy val httpClient = "org.apache.httpcomponents" % "httpclient" % Version.httpcomponents - lazy val akkaKafkaStreams = "com.typesafe.akka" %% "akka-stream-kafka" % Version.alpakkaKafka - lazy val embeddedKafka = "io.github.embeddedkafka" %% "embedded-kafka" % Version.embeddedKafka - lazy val alpakkaS3 = "com.lightbend.akka" %% "akka-stream-alpakka-s3" % Version.alpakkaS3 - lazy val akkaQuartzScheduler = "com.enragedginger" %% "akka-quartz-scheduler" % Version.akkaQuartzScheduler - lazy val sbtResolver = "io.spray" %% "sbt-revolver" % Version.sprayresolver - lazy val enumeratum = "com.beachape" %% "enumeratum" % Version.enumeratum - lazy val enumeratumCirce = "com.beachape" %% "enumeratum-circe" % Version.enumeratumCirce - lazy val monix = "io.monix" %% "monix" % Version.monix - lazy val lettuce = "io.lettuce" % "lettuce-core" % Version.lettuce - lazy val scalaJava8Compat = "org.scala-lang.modules" %% "scala-java8-compat" % Version.java8Compat - lazy val guava = "com.google.guava" % "guava" % Version.guava - lazy val awsSesSdk = "com.amazonaws" % "aws-java-sdk-ses" % Version.awsSesSdk exclude ("commons-logging", "commons-logging") - lazy val zeroAllocationHashing = "net.openhft" % "zero-allocation-hashing" % Version.zeroAllocation - lazy val cormorant = "io.chrisdavenport" %% "cormorant-parser" % Version.cormorant - lazy val cormorantGeneric = "io.chrisdavenport" %% "cormorant-generic" % Version.cormorant - lazy val alpakkaFile = "com.lightbend.akka" %% "akka-stream-alpakka-file" % Version.alpkakkaFile - lazy val scalacheckShapeless = "com.github.alexarchambault" %% "scalacheck-shapeless_1.14" % Version.scalacheckShapeless % Test - lazy val diffx = "com.softwaremill.diffx" %% "diffx-core" % Version.diffx % Test - lazy val kubernetesApi = "io.kubernetes" % "client-java" % Version.kubernetesApi - - // overriding the log4j-slf4j bridge used by spring, transitively brought in by s3mock - // this is needed because of CVE-2021-44228 https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-44228 - lazy val log4jToSlf4j = "org.apache.logging.log4j" % "log4j-to-slf4j" % Version.log4j % Test -} \ No newline at end of file