Powered By Blogger

Tuesday, October 8, 2019

Creating Dataframe of structtype

val dfLocal2 = Seq(
      (1,"PRESCHOOL", true, "basan", "11", "asdas", "3", "LINES"),
      (2, null,        false, "basan", "32", "asfasf", "13", "/COSMETICS"),
      (3,"FURNITURE",         null, "basan", "23", "asfasf HOME", "4", "HOME")

    ).toDF("id_int_null","string_null","boolean_null","TCIN","DIVISION_ID","DIVISION_NAME","GROUP_ID","GROUP_NAME")
dfLocal2.show()
dfLocal2.printSchema()



___


import org.apache.spark.sql.Row
import org.apache.spark.sql._
import org.apache.spark.sql.types._

val someData = Seq(
  Row(8, "bat"),
  Row(64, "mouse"),
  Row(-27, "horse"),
  Row(null,"abc")
)

val someSchema = List(
  StructField("number", IntegerType, true),
  StructField("word", StringType, true)
)

val someDF = spark.createDataFrame(
  sc.parallelize(someData),
  StructType(someSchema)
)

someDF.show(6)

someDF.show()

No comments:

Post a Comment