Powered By Blogger

Sunday, October 13, 2019

Date Time Operation in Scala and Spark


scala> import java.sql.Date
import java.sql.Date

scala> import org.apache.spark.sql.types.{DateType, IntegerType}
import org.apache.spark.sql.types.{DateType, IntegerType}

scala> Date.valueOf("2016-09-30")
res0: java.sql.Date = 2016-09-30

scala> import java.time.LocalDateTime
import java.time.LocalDateTime

scala>

scala> import java.sql.Timestamp
import java.sql.Timestamp

scala> val levels = Seq(
     |   // (year, month, dayOfMonth, hour, minute, second)
     |   ((2012, 12, 12, 12, 12, 12), 5),
     |   ((2012, 12, 12, 12, 12, 14), 9),
     |   ((2012, 12, 12, 13, 13, 14), 4),
     |   ((2016, 8,  13, 0, 0, 0), 10),
     |   ((2017, 5,  27, 0, 0, 0), 15)).
     |   map { case ((yy, mm, dd, h, m, s), a) => (LocalDateTime.of(yy, mm, dd, h, m, s), a) }.
     |   map { case (ts, a) => (Timestamp.valueOf(ts), a) }.
     |   toDF("time", "level")
levels: org.apache.spark.sql.DataFrame = [time: timestamp, level: int]

scala>

scala> levels.show
+-------------------+-----+
|               time|level|
+-------------------+-----+
|2012-12-12 12:12:12|    5|
|2012-12-12 12:12:14|    9|
|2012-12-12 13:13:14|    4|
|2016-08-13 00:00:00|   10|
|2017-05-27 00:00:00|   15|
+-------------------+-----+


scala>

https://jaceklaskowski.gitbooks.io/mastering-spark-sql/spark-sql-functions-datetime.html

No comments:

Post a Comment