1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
| import org.apache.hadoop.fs.{FileAlreadyExistsException, FileSystem, FileUtil, Path}
val srcFileSystem: FileSystem = FileSystemUtil
.apply(spark.sparkContext.hadoopConfiguration)
.getFileSystem(sourceFile)
val dstFileSystem: FileSystem = FileSystemUtil
.apply(spark.sparkContext.hadoopConfiguration)
.getFileSystem(sourceFile)
FileUtil.copy(
srcFileSystem,
new Path(new URI(sourceFile)),
dstFileSystem,
new Path(new URI(targetFile)),
true,
spark.sparkContext.hadoopConfiguration)
|