NoSuchMethodError Внутреннее ведение журнала Spark

Я упаковал свое приложение в файл jar, однако, когда я пытаюсь его выполнить, приложение завершается с ошибкой:

Exception in thread "main" java.lang.NoSuchMethodError: org.apache.spark.internal.Logging.$init$(Lorg/apache/spark/internal/Logging;)V
    at org.apache.spark.streaming.kafka010.KafkaUtils$.<init>(KafkaUtils.scala:39)
    at org.apache.spark.streaming.kafka010.KafkaUtils$.<clinit>(KafkaUtils.scala)
    at jobs.SparkJobExample$.main(SparkJobExample.scala:56)
    at jobs.SparkJobExample.main(SparkJobExample.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
    at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:846)
    at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)
    at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:194)
    at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
    at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:921)
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:932)
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

Строка SparkJobExample.scala:56, указанная в этой ошибке, выглядит так:

 val dataRDD = KafkaUtils.createRDD(sparkContext,kafkaParams(),offsetRanges,LocationStrategies.PreferConsistent)

Как я понял, это может быть проблема с версией зависимостей, я проверил их, но все в порядке. Что может быть причиной этой ошибки?

Содержимое моего файла build.sbt:

name := "SparkK8s"
version := "0.1"
scalaVersion := "2.12.8"
val sparkVersion = "2.4.0"
val circeVersion = "0.11.0"

dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-core" % "2.9.8"
dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.9.8"
dependencyOverrides += "com.fasterxml.jackson.module" % "jackson-module-scala" % "2.9.8"

resolvers += "Spark Packages Repo" at "http://dl.bintray.com/spark-packages/maven"
resolvers += "confluent" at "http://packages.confluent.io/maven/"

libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % sparkVersion,
  "org.apache.spark" %% "spark-sql" % sparkVersion,
  "org.apache.spark" %% "spark-streaming" % sparkVersion,
  "org.apache.spark" %% "spark-hive" % sparkVersion,
  "org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion,
  "org.apache.spark" %% "spark-streaming-kafka-0-10_2.12" % sparkVersion,
  "org.apache.kafka" %% "kafka" % "2.1.0",
  "org.apache.kafka" % "kafka-log4j-appender" % "2.1.0",
  "org.scalatest" %% "scalatest" % "3.2.0-SNAP10" % "it, test",
  "org.scalacheck" %% "scalacheck" % "1.14.0" % "it, test",
  "io.kubernetes" % "client-java" % "3.0.0" % "it",
  "org.json" % "json" % "20180813",
  "io.circe" %% "circe-core" % circeVersion,
  "io.circe" %% "circe-generic" % circeVersion,
  "io.circe" %% "circe-parser" % circeVersion,
  "org.apache.avro" % "avro" % "1.8.2",
  "io.confluent" % "kafka-avro-serializer" % "5.0.1"
)

ОБНОВЛЕНИЕ: я развернул файл jar как с Kubernetes, так и локально. Результат был таким же

$SPARK_HOME/bin/spark-submit \
   --master local \
   --name spark-etl-job \
   --class jobs.SparkJobExample \
   --conf spark.executor.instances=1 \
http://192.168.99.100:8089/spark_hive_job.jar <params_list>

Настройки сборки SBT:

assemblyJarName in assembly := "spark_hive_job.jar"

mainClass in assembly := Some("jobs.SparkJobExample")

test in assembly := {}

assemblyMergeStrategy in assembly := {
  case x if x.endsWith(".conf") => MergeStrategy.discard
  case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") => MergeStrategy.first
  case PathList("org", "apache", "commons", "logging", _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "beanutils", _*) => MergeStrategy.first
  case PathList("org", "apache", "commons", "collections", _*) => MergeStrategy.first
  case PathList("org", "apache", "hadoop", "yarn", _*) => MergeStrategy.first
  case PathList("org", "aopalliance", _*) => MergeStrategy.first
  case PathList("org", "objenesis", _*) => MergeStrategy.first
  case PathList("com", "sun", "jersey", _*) => MergeStrategy.first
  case PathList("org", "apache", "hadoop", "yarn", _*) => MergeStrategy.first
  case PathList("org", "slf4j", "impl", _*) => MergeStrategy.first
  case PathList("com", "codahale", "metrics", _*) => MergeStrategy.first
  case PathList("javax", "transaction", _*) => MergeStrategy.first
  case PathList("javax", "inject", _*) => MergeStrategy.first
  case PathList("javax", "xml", _*) => MergeStrategy.first
  case PathList("META-INF", "jersey-module-version") => MergeStrategy.first
  case PathList("example",  _*) => MergeStrategy.discard
  case PathList("utils",  _*) => MergeStrategy.discard
  case PathList("data_generation",  _*) => MergeStrategy.discard
  case PathList("example_tests",  _*) => MergeStrategy.discard
  case PathList("plugin.xml") => MergeStrategy.first
  case PathList("parquet.thrift") => MergeStrategy.first
  case PathList("git.properties") => MergeStrategy.first
  case PathList("codegen", "config.fmpp") => MergeStrategy.first
  case PathList("git.properties") => MergeStrategy.first
  case PathList("overview.html") => MergeStrategy.discard
  case x => (assemblyMergeStrategy in assembly).value(x)
}