diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ab4b9a2ad1d303df99a907eda5e07503df93ec9..d60cfb0f9d3d6868f357319417f9c366df1dd139 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ build-repo: - tags script: - sbt publish - - mv /builds/monitoring/spark-monit-utilities/target/scala-2.11/ivy-${CI_COMMIT_TAG}.xml /builds/monitoring/spark-monit-utilities/release/monit-utilities/monit-utilities_2.11/${CI_COMMIT_TAG} + - mv /builds/monitoring/spark-monit-utilities/target/scala-2.12/resolution-cache/monit-utilities/monit-utilities_2.12/${CI_COMMIT_TAG}/resolved.xml.xml /builds/monitoring/spark-monit-utilities/release/monit-utilities/monit-utilities_2.12/${CI_COMMIT_TAG}/ivy-${CI_COMMIT_TAG}.xml artifacts: expire_in: 10 yrs paths: diff --git a/build.sbt b/build.sbt index 7a7474f5890ce62a9e30efc5ef41a00a5fd51eb1..c659defd05c2e610755d5a83536ac396b90f8450 100644 --- a/build.sbt +++ b/build.sbt @@ -1,25 +1,21 @@ name := "monit-utilities" -version := "3.1" +version := "3.4" -scalaVersion := "2.11.11" +scalaVersion := "2.12.17" -val sparkVersion = "2.3.2" +val sparkVersion = "3.4.0" libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % sparkVersion, "org.apache.spark" %% "spark-sql" % sparkVersion, "org.apache.spark" %% "spark-streaming" % sparkVersion, "org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion, - "com.github.scopt" %% "scopt" % "3.7.0" + "com.github.scopt" %% "scopt" % "4.0.1" ) resolvers ++= Seq( - "Spray Repository" at "http://repo.spray.cc/", "Cloudera Repository" at "https://repository.cloudera.com/artifactory/cloudera-repos/", - "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/", - "Second Typesafe repo" at "http://repo.typesafe.com/typesafe/maven-releases/", - "Mesosphere Public Repository" at "http://downloads.mesosphere.io/maven", Resolver.sonatypeRepo("public") ) diff --git a/project/assembly.sbt b/project/assembly.sbt index becafdfceef5a0ac48ddaeb65a6b6cf18c76228f..5b205051d3e95e4525a76801e695ef15e3c704ed 100644 --- a/project/assembly.sbt +++ b/project/assembly.sbt @@ -1,3 +1 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.2") - -resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns) +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") \ No newline at end of file diff --git a/project/build.properties b/project/build.properties index e0cbc71da47b5adef7963bcd21fd7fe184f2ba56..63527fe119278bb010de5b7f42d1067df2effe81 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 0.13.13 \ No newline at end of file +sbt.version = 1.5.1 \ No newline at end of file diff --git a/src/main/scala/ch/cern/monitoring/utilities/io/KafkaUtils.scala b/src/main/scala/ch/cern/monitoring/utilities/io/KafkaUtils.scala index dfe7d57d49beecedae7761d5f4747e4757808ee0..860e462029b5b7822b634b39f1ce6501bd79aad0 100644 --- a/src/main/scala/ch/cern/monitoring/utilities/io/KafkaUtils.scala +++ b/src/main/scala/ch/cern/monitoring/utilities/io/KafkaUtils.scala @@ -16,6 +16,7 @@ case class KafkaUtils(spark: SparkSession, inputBrokers: String, outputBrokers: keyStoreLocation: String, keyStorePassword: String, keyPassword: String, logger: Logger) { this(spark, inputBrokers, outputBrokers, logger, Map( + "kafka.enable.idempotence" -> false.toString, "kafka.ssl.truststore.location" -> trustStoreLocation, "kafka.ssl.truststore.password" -> trustStorePassword, "kafka.ssl.keystore.location" -> keyStoreLocation,