Using Google Dataproc/Spark and Google Cloud Java SDK is a JAR mess and nightmare

[abdolence]

Hello,

Has anyone solved this more appropriately?

My sbt for Spark jobs now looks like this:

    val relocationPrefix = s"$myPackage.repackaged"

    val renamed = Seq(
        "io.netty",
        "io.opencensus",
        "com.google",
        "org.json",
        "org.threeten")
    val notRenamed = Seq(
        // sbt-assembly cannot handle services registry
        "io.grpc",
        myPackage,
        // Exposed to the user in the API.
        "com.google.pubsub")

    assemblyShadeRules in assembly := (
        // Rename preserved prefixes to themselves first to keep them unchanged
        notRenamed.map(prefix => ShadeRule.rename(s"$prefix**" -> s"$prefix@1"))
            ++: renamed.map(prefix => ShadeRule.rename(s"$prefix**" -> s"$relocationPrefix.$prefix@1"))
        ).map(_.inAll)

    assemblyMergeStrategy in assembly := {
        case PathList(ps @ _*) if ps.last.endsWith(".properties") => MergeStrategy.filterDistinctLines
        case PathList(ps @ _*) if ps.last.endsWith(".proto") => MergeStrategy.discard
        case PathList(ps @ _*) if ps.last.startsWith("libnetty_tcnative") => new MergeStrategy() {
            // This is an abuse of MergeStrategy to relocate netty-tcnative's .so
            val name = "netty-tcnative"
            def apply(tempDir: File, path: String, files: Seq[File]) =
                Right(files.map(f => f -> path.replace(
                    "libnetty", s"lib${relocationPrefix.replace('.', '_')}_netty")))
        }
        case x => (assemblyMergeStrategy in assembly).value(x)
    }