SBT hacks

Skipping test cases in build

  sbt 'set test in assembly := {}' clean assembly

  sbt 'set test in assembly := {}' clean package

Referring to the local jar in sbt

unmanagedJars in Compile += file("/Users/basan/Documents/Projects/application/application-core/target/scala-2.11/application-core-assembly-1.8.2.jar")

https://www.scala-sbt.org/1.x/docs/Library-Management.html

unmanagedBase := baseDirectory.value / "custom_lib"

https://github.com/sbt/sbt-assembly/blob/master/src/sbt-test/sbt-assembly/deps/build.sbt







lazy val root = (project in file(".")).
	settings(
	version := "0.1",
	scalaVersion := "2.11.12",
	libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.1" % "test",
	libraryDependencies += "ch.qos.logback" % "logback-classic" % "0.9.29" % "runtime",
	unmanagedJars in Compile ++= {
	(baseDirectory.value / "lib" / "compile" ** "*.jar").classpath
	},
	unmanagedJars in Runtime ++= {
	(baseDirectory.value / "lib" / "runtime" ** "*.jar").classpath
	},
	unmanagedJars in Test ++= {
	(baseDirectory.value / "lib" / "test" ** "*.jar").classpath
	},
	assemblyExcludedJars in assembly := {
	(fullClasspath in assembly).value filter {_.data.getName == "compile-0.1.0.jar"}
	},
	assemblyJarName in assembly := "foo.jar",
	TaskKey[Unit]("check") := {
	val process = sys.process.Process("java", Seq("-jar", (crossTarget.value / "foo.jar").toString))
	val out = (process!!)
	if (out.trim != "hello") sys.error("unexpected output: " + out)
	()
	}
	)

Typesafe config map

app {
  #application name
  name: "Streaming"

  dataportal {
        timelinessKafka {
           topicName = "kelsa-DIP-replace"
           producerDetails {
           bootstrap.servers = "broker:9093"
           client.id = "application name"
           key.serializer="org.apache.kafka.common.serialization.StringSerializer"
           value.serializer="org.apache.kafka.common.serialization.StringSerializer"
           ssl.truststore.location = "Data/certificates/metakafka/client.truststore_stg.jks"
           ssl.keystore.location = "Data/certificates/metakafka/server.keystore_stg.jks"
           security.protocol = "SSL"
           ssl.protocol = "TLSv1.2"
           ssl.endpoint.identification.algorithm = ""
           }
        }
    }
}

spark-shell --packages com.typesafe:config:1.3.3

import com.typesafe.config._
import java.io.File
import scala.collection.JavaConverters._


val path = "/Users/basan/Documents/Projects/metricPush2/application.conf"
val config =ConfigFactory.parseFile(new File(path))

val producerDetails = config.getConfig("app.dataportal.timelinessKafka.producerDetails")
producerDetails.getString("bootstrap.servers")


producerDetails
      .entrySet()
      .asScala
      .map( entry => entry.getKey -> entry.getValue.unwrapped().toString ).toMap

res7: scala.collection.immutable.Map[String,String] = Map(ssl.truststore.location -> Data/certificates/metakafka/client.truststore_stg.jks, security.protocol -> SSL, bootstrap.servers -> broker:9093, client.id -> application name, value.serializer -> org.apache.kafka.common.serialization.StringSerializer, ssl.protocol -> TLSv1.2, ssl.endpoint.identification.algorithm -> "", key.serializer -> org.apache.kafka.common.serialization.StringSerializer, ssl.keystore.location -> Data/certificates/metakafka/server.keystore_stg.jks)


config
      .entrySet()
      .asScala
      .map( entry => entry.getKey -> entry.getValue.toString ).toMap

res5: scala.collection.immutable.Map[String,String] = Map(app.dataportal.timelinessKafka.producerDetails.ssl.keystore.location -> Quoted("Data/certificates/metakafka/server.keystore_stg.jks"), app.contactDetails -> Quoted("DataSciences-DE-GSCL-DigitalFulfillment@Target.com"), app.teamName -> Quoted("GSCL-Digital Fulfillment"), app.dataportal.timelinessKafka.producerDetails.key.serializer -> Quoted("org.apache.kafka.common.serialization.StringSerializer"), app.dataportal.timelinessKafka.producerDetails.value.serializer -> Quoted("org.apache.kafka.common.serialization.StringSerializer"), app.dataportal.timelinessKafka.producerDetails.bootstrap.servers -> Quoted("broker:9093"), app.kelsaApplicationPath -> Quoted("Data/KelsaPath/"), app.dataportal.timelinessKafka.producerDetails.client.id -...