我在下面的代码中尝试创建一个Spark数据帧,其中的字段是一个struct。我应该替换什么
???
为了让这个工作。
import org.apache.spark.sql.types._
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
val spark: SparkSession = SparkSession.builder()
.appName("NodesLanesTest")
.getOrCreate()
val someData = Seq(
Row(1538161836000L, 1538075436000L, "cargo3", 3L, ???("Chicago", "1234"))
)
val someSchema = StructType(
List(
StructField("ata", LongType, nullable = false),
StructField("atd", LongType, nullable = false),
StructField("cargo", StringType, nullable = false),
StructField("createdDate", LongType, nullable = false),
StructField("destination",
StructType(List(
StructField("name", StringType, nullable = false),
StructField("uuid", StringType, nullable = false)
))))
val someDF = spark.createDataFrame(
spark.sparkContext.parallelize(someData),
StructType(someSchema)
)