spark sql java api_spark的javaAPI,对Json读取和操作,视图创建和sql的操作

import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}

import org.apache.spark.{SparkConf, SparkContext}objectDome {/**

* 单词统计*/def fun2: Unit={

val conf= newSparkConf()

conf.setAppName("wordCount")

conf.setMaster("local")

val sc= newSparkContext(conf)

val line= sc.textFile("E:\\words.txt")

line.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).sortBy(_._2,true).foreach(println(_))

sc.stop()

}/**

*从hdfs上读取,需要打成jar包*/def fun3(args:Array[String]): Unit={

val conf= new SparkConf().setAppName("WC")

val sc= newSparkContext(conf)

val line= sc.textFile(args(0))

line.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).sortBy(_._2,false).saveAsTextFile(args(1))

sc.stop()

}/**

* 对统计相邻单词出现的个数

* A;B;C;D;E;F

* C;A;D;C;E;S

* C;E;F*/def fun(): Unit={

val conf= newSparkConf()

conf.setAppName("wordCount")

conf.setMaster("local")

val sc= newSparkContext(conf)

val line= sc.textFile("E:\\words.txt")

line.map(_.split(";")).flatMap(x=>{for(i

}).foreach(println(_))

sc.stop()

}/**

* 读取JSON格式的内容

* 1.创建SparkConf

* 2.创建SparkContext

* 3.得到SqlContext

* 4.得到DataFrame

* 5.显示DataFrame中的数据*/def fun1(): Unit={

val conf= new SparkConf().setAppName("JSON").setMaster("local")

val sc= newSparkContext(conf)

val sqlContext= newSQLContext(sc)

val people= sqlContext.read.format("JSON").load("E:\\user.json")

people.show(30,100)//num 读取前面多山行 truncat显示列宽

sc.stop()

}/**

* 不过时方法,参考官网*/def fun4(): Unit={

val conf= new SparkConf().setAppName("JSON").setMaster("local")

val spark= SparkSession.builder().appName("JSONDome")

.config(conf).getOrCreate()

val people:DataFrame= spark.read.json("E:\\user.json")

people.show(4,100)

people.printSchema()//显示表结构

people.select("name").show()

import spark.implicits._//条件查询需要将数据进行隐式转换

people.select($"name", $"age"+10).show()

people.select($"name", $"age">15).show()

people.select("name","age").filter( $"age">15).show()

people.groupBy("age").count().show() //返回DataFrame对象

spark.stop()

}/**

* 生成临时表,使用SQL查询语句查询*/def fun5(): Unit={

val conf= new SparkConf().setAppName("JSON").setMaster("local")

val spark= SparkSession.builder().appName("JSONDome")

.config(conf).getOrCreate()

val people:DataFrame= spark.read.json("E:\\user.json")//createGlobalTempView是临时的视图

people.createOrReplaceTempView("user") //创建//sql返回的还是DataFrame

spark.sql("select * from user").show()

spark.sql("select id,name,age,balance yue from user where age>15").show() //起别名不能用中文//spark.sql("update set age = age+100 from user")//不能对表中数据进行修改

spark.sql("select * from user").show()

people.createGlobalTempView("user1") //创建的是临时表,如果表名与user相同,会覆盖之前一个表

spark.sql("select * from global_temp.user1").show()

spark.newSession().sql("select * from global_temp.user1").show() //跨会话框操作

spark.sql("select id,name,age,balance yue from global_temp.user1 where age>15").show() //起别名不能用中文

people.createTempView("user2") //使用这个创建不会覆盖之前参 存在的表

spark.sql("select * from user2").show()//spark.sql("alter table user2 drop COLUMN balance RESTRICT")//不能修改表结构

spark.sql("select * from user2").show()

spark.stop()

}/**

* 读取多个文件*/def fun6(): Unit={

val conf= new SparkConf().setAppName("JSON").setMaster("local")

val spark= SparkSession.builder().appName("JSONDome")

.config(conf).getOrCreate()

val people:DataFrame= spark.read.json("E:\\empAndDept/*.json")

people.show(100)

}

def main(args: Array[String]): Unit={

fun6()

}

}