ita
06/10/2022, 2:30 PMUser class threw exception: java.lang.NoSuchMethodError: org.apache.spark.sql.catalyst.expressions.objects.Invoke$.apply$default$5()Z
at org.apache.spark.sql.KotlinReflection$.$anonfun$serializerFor$16(KotlinReflection.scala:754)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
at org.apache.spark.sql.KotlinReflection$.$anonfun$serializerFor$1(KotlinReflection.scala:748)
at scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:73)
at org.apache.spark.sql.KotlinReflection.cleanUpReflectionObjects(KotlinReflection.scala:1012)
at org.apache.spark.sql.KotlinReflection.cleanUpReflectionObjects$(KotlinReflection.scala:1011)
at org.apache.spark.sql.KotlinReflection$.cleanUpReflectionObjects(KotlinReflection.scala:47)
at org.apache.spark.sql.KotlinReflection$.serializerFor(KotlinReflection.scala:591)
at org.apache.spark.sql.KotlinReflection$.serializerFor(KotlinReflection.scala:578)
at org.apache.spark.sql.KotlinReflection.serializerFor(KotlinReflection.scala)
at org.jetbrains.kotlinx.spark.api.ApiV1Kt.kotlinClassEncoder(ApiV1.kt:180)
at org.jetbrains.kotlinx.spark.api.ApiV1Kt.generateEncoder(ApiV1.kt:167)
at movoto.leadsync.jobs.ExtractJob.<init>(ExtractJob.kt:218)
at movoto.leadsync.ExtractCommand.run(SparkJobMain.kt:104)
at com.github.ajalt.clikt.parsers.Parser.parse(Parser.kt:198)
at com.github.ajalt.clikt.parsers.Parser.parse(Parser.kt:211)
at com.github.ajalt.clikt.parsers.Parser.parse(Parser.kt:18)
at com.github.ajalt.clikt.core.CliktCommand.parse(CliktCommand.kt:395)
at com.github.ajalt.clikt.core.CliktCommand.parse$default(CliktCommand.kt:392)
at com.github.ajalt.clikt.core.CliktCommand.main(CliktCommand.kt:410)
at com.github.ajalt.clikt.core.CliktCommand.main(CliktCommand.kt:435)
at movoto.leadsync.SparkJobMainKt.main(SparkJobMain.kt:189)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:740)
Unmanaged Application:
Ilya Muradyan
06/10/2022, 2:42 PMJolan Rensen [JetBrains]
06/10/2022, 2:45 PMita
06/10/2022, 2:52 PMPasha Finkelshteyn
06/10/2022, 2:58 PMita
06/10/2022, 3:01 PMkotlin-spark-api-3.0:1.0.2
but local testing produced the same issue w/ kotlin-spark-api-3.2:1.1.0
on Spark 3.2.0 but not Spark 3.2.1Pasha Finkelshteyn
06/10/2022, 3:02 PMita
06/10/2022, 3:02 PMPasha Finkelshteyn
06/10/2022, 3:03 PMJolan Rensen [JetBrains]
06/10/2022, 3:07 PMita
06/10/2022, 3:12 PMPasha Finkelshteyn
06/10/2022, 3:13 PMJolan Rensen [JetBrains]
06/10/2022, 3:13 PMita
06/10/2022, 3:30 PMimport io.kotest.core.spec.style.FunSpec
import org.apache.spark.sql.Dataset
import org.apache.spark.sql.SparkSession
import org.jetbrains.kotlinx.spark.api.dsOf
import <http://org.jetbrains.kotlinx.spark.api.to|org.jetbrains.kotlinx.spark.api.to>
data class DowncastType(
val intType: Int? = null,
)
class SparkDowncastTest : FunSpec({
val spark = SparkSession.builder().master("local[*]").orCreate
afterSpec {
spark.stop()
}
test("test spark downcasting") {
val dataset = spark.dsOf("intType" to 1)
val casted: Dataset<DowncastType> = <http://dataset.to|dataset.to>()
}
})
Jolan Rensen [JetBrains]
06/13/2022, 9:44 AMException in thread "main" org.apache.spark.sql.AnalysisException: cannot resolve 'intType' given input columns: [first, second]
Which makes sense, since you are trying to implicitly convert a Dataset<Pair<String, Int>>
to Dataset<DowncastType>
which is a completely different classdata class Type(val intType: Int)
...
dsOf(Type(intType = 1))
...
instead 🙂ita
06/14/2022, 4:39 PM