Spark RDD 复杂算子

最新推荐文章于 2025-05-13 12:09:08 发布

原创最新推荐文章于 2025-05-13 12:09:08 发布 · 366 阅读

0 ·

本内容遵循CC 4.0 BY-SA版权协议

标签

#python

本文深入探讨了Spark中RDD的Transformation操作如map、flatMap、filter等，以及Action操作如collect、aggregate、saveAsTextFile等，解析了它们在大数据处理中的应用与区别。

aggregateByKey 是Transformation
reduceByKey 是Transformation
filter 是Transformation
flatMap 是Transformation
map 是ransformation
mapPartition 是ransformation
mapPartitionWithIndex 是ransformation

collect 是Action
aggregate 是Action
saveAsTextFile 是Action
foreach 是Action
foreachPartition 是Action

mapPartitionsWithIndex
可以看到每个元素被分配到哪个分区了

val func = (index: Int, iter: Iterator[Int]) => {
  iter.map(x => "[partID:" +  index + ", val: " + x + "]")
}
val rdd1 = sc.parallelize(List(1,2,3), 2)
rdd1.mapPartitionsWithIndex(func).collect

aggregate

def func1(index: Int, iter: Iterator[(Int)]) : Iterator[String] = {
  iter.toList.map(x => "[partID:" +  index + ", val: " + x + "]").iterator
}
val rdd1 = sc.parallelize(List(1,2,3,4,5,6,7,8,9), 2)
rdd1.mapPartitionsWithIndex(func1).collect
rdd1.aggregate(0)(_+_, _ + _)  # 科里第一个参数是起始值分片聚合前用，总聚合时也用，第二个括号，第一个参数是组内求和，第二个参数是组之间求和
rdd1.aggregate(0)(math.max(_, _), _ + _)  # 组内求最大值（就是求两个"_"的最大值）
rdd1.aggregate(5)(math.max(_, _), _ + _)


val rdd2 = sc.parallelize(List("a","b","c","d","e","f"),2)
def func2(index: Int, iter: Iterator[(String)]) : Iterator[String] = {
  iter.toList.map(x => "[partID:" +  index + ", val: " + x + "]").iterator
}
rdd2.aggregate("")(_ + _, _ + _)
rdd2.aggregate("=")(_ + _, _ + _) # =号会被使用三次，分组时每组各使用一次，聚合时再使用一次

val rdd3 = sc.parallelize(List("12","23","345","4567"),2)
rdd3.aggregate("")((x,y) => math.max(x.length, y.length).toString, (x,y) => x + y)

val rdd4 = sc.parallelize(List("12","23","345",""),2)
rdd4.aggregate("")((x,y) => math.min(x.length, y.length).toString, (x,y) => x + y)

val rdd5 = sc.parallelize(List("12","23","","345"),2)
rdd5.aggregate("")((x,y) => math.min(x.length, y.length).toString, (x,y) => x + y)

aggregateByKey
按key进行分组计算

val pairRDD = sc.parallelize(List( ("cat",2), ("cat", 5), ("mouse", 4),("cat", 12), ("dog", 12), ("mouse", 2)), 2)
def func2(index: Int, iter: Iterator[(String, Int)]) : Iterator[String] = {
  iter.map(x => "[partID:" +  index + ", val: " + x + "]")
}
pairRDD.mapPartitionsWithIndex(func2).collect
pairRDD.aggregateByKey(0)(math.max(_, _), _ + _).collect
pairRDD.aggregateByKey(100)(math.max(_, _), _ + _).collect

collectAsMap、

val rdd = sc.parallelize(List(("a", 1), ("b", 2)))
rdd.collectAsMap

combineByKey

val rdd1 = sc.textFile("hdfs://node-1.edu360.cn:9000/wc").flatMap(_.split(" ")).map((_, 1))
val rdd2 = rdd1.combineByKey(x => x, (a: Int, b: Int) => a + b, (m: Int, n: Int) => m + n)
rdd2.collect

val rdd3 = rdd1.combineByKey(x => x + 10, (a: Int, b: Int) => a + b, (m: Int, n: Int) => m + n)
rdd3.collect


val rdd4 = sc.parallelize(List("dog","cat","gnu","salmon","rabbit","turkey","wolf","bear","bee"), 3)
val rdd5 = sc.parallelize(List(1,1,2,2,2,1,2,2,2), 3)
val rdd6 = rdd5.zip(rdd4)
val rdd7 = rdd6.combineByKey(List(_), (x: List[String], y: String) => x :+ y, (m: List[String], n: List[String]) => m ++ n)

countByKey

val rdd1 = sc.parallelize(List(("a", 1), ("b", 2), ("b", 2), ("c", 2), ("c", 1)))
rdd1.countByKey
rdd1.countByValue

filterByRange

val rdd1 = sc.parallelize(List(("e", 5), ("c", 3), ("d", 4), ("c", 2), ("a", 1)))
val rdd2 = rdd1.filterByRange("b", "d")
rdd2.colllect

flatMapValues

val a = sc.parallelize(List(("a", "1 2"), ("b", "3 4")))
rdd3.flatMapValues(_.split(" "))

foldByKey

val rdd1 = sc.parallelize(List("dog", "wolf", "cat", "bear"), 2)
val rdd2 = rdd1.map(x => (x.length, x))
val rdd3 = rdd2.foldByKey("")(_+_)

val rdd = sc.textFile("hdfs://node-1.edu360.cn:9000/wc").flatMap(_.split(" ")).map((_, 1))
rdd.foldByKey(0)(_+_)

foreach foreachPartition

rdd1.foreachPartition(it => it.foreach(x => println(x * 1000)))

combineByKey

val rdd1 = sc.parallelize(List(("hello",2),("jerry",3),("hello",4),("jerry",1)),2)
val rdd2 = rdd1.combineByKey(x => x, (a: Int, b: Int) => a + b, (m: Int, n: Int) => m + n)
rdd2.collect