access HDFS from Spark

HDFS from Spark.md

val conf = sc.hadoopConfiguration
val fs = org.apache.hadoop.fs.FileSystem.get(conf)
val exists = fs.exists(new org.apache.hadoop.fs.Path("/path/on/hdfs")) // File or directory

val sequenceFiles1 = htu.getDFSCluster.getFileSystem.listStatus(new Path(outputFileDirforCrashes)).filter(_.isDirectory).map(_.getPath.toString)
        sequenceFiles1.foreach(dirPath=>{
            val files = htu.getDFSCluster.getFileSystem.listStatus(new Path(dirPath)).filter(_.isFile).map(_.getPath.toString)
            files.foreach(filePath=>{
                print(filePath)
            })
        })