1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
| object CreateHfile {
def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("CreateHfile").setMaster(args(0))
val sc = new SparkContext(conf)
val hbaseConf = HBaseConfiguration.create()
val rdd = sc.textFile(args(1))
.flatMap(v =>{
val x = new javaList[String]()
for( a <- 1 to 9999){
x.add(v + "%04d".format(a))
}
x.toArray
}
)
.sortBy(v=>v.toString)
.map(r =>(
new ImmutableBytesWritable(Bytes.toBytes(r.toString)),
new KeyValue(
Bytes.toBytes(r.toString), Bytes.toBytes("phoneFamliy"), Bytes.toBytes("phoneCol"),
System.currentTimeMillis(),
KeyValue.Type.DeleteColumn)
))
rdd.saveAsNewAPIHadoopFile(args(2), classOf[ImmutableBytesWritable],classOf[KeyValue],classOf[HFileOutputFormat2], hbaseConf)
sc.stop()
}
}
|