Powered By Blogger

Monday, May 3, 2021

Spark Read content of the file as is

 json file


{
"sharedObjectName": "sharedObjectName",
"writeInProgress":false,
"tableName":"item",
"lastSyncTimeInEpoch":1619783310,
"latestLocation":"latestLocation/latestLocation/latestLocation",
"oneStepLagLocation":"oneStepLagLocation/oneStepLagLocation/oneStepLagLocation"
}



Spark Code to read the file

@throws[ConfigException]
def readJSONFileAsIs( location: String)(sparkSession:SparkSession): String ={
logInfo(s"Read the file contents as is from the location $location")
try {
val textFileDataFrame : Dataset[String] = sparkSession.read.textFile(location)
val builder : StringBuilder = new StringBuilder()
textFileDataFrame.collect().foreach(line=>{
builder.append(line)
})
val fileContents : String = builder.toString()
logInfo(s"Content of the file $fileContents")
fileContents

}catch {
case exception: Exception => logError(s"Error in reading the file from the location $location",exception)
throw ConfigException(s"Error in reading the file from the location $location")
}

}

No comments:

Post a Comment