hadoop获取job信息，maptask，reducetask获取信息，hadoop监控job执行状况（cdh4.2 ）

最近在做hadoop的job监控，需要自己写代码，获取job，map和reduce的信息，参照API和网上的资料，实现了job信息的获取，代码如下：

package hadoop;

//java工程中，将hadoop的包导进来，运行此代码，chd4.2 hadoop环境

//下面的包都在hadoop中，自己找一下即可，java包不用管，报错点一下，自动加进来

import java.io.BufferedReader;

import java.io.InputStreamReader;

import java.net.InetSocketAddress;

import java.net.URL;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Locale;

import java.util.regex.Pattern;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.mapred.ClusterStatus;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.TaskReport;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.mapred.JobStatus;

public class Job extends Configured implements Tool{   

   

public static final String HOST="master";//主机名称

public static final int JOBTRACKER_PORT=8021;//jobtracker rpc 端口

public static final String NAMENODE_PORT="cdh4";//hdfs 地址

         

     

    public static void main(String[] args) throws Exception{  

     

    System.out.println("The name of task:"); 

        InetSocketAddress jobTracker=new InetSocketAddress(HOST,JOBTRACKER_PORT);//远程连接jobtracker

    JobClient jobClient=new JobClient(jobTracker, getsConf());//初始化jobclient

    JobStatus[] jobStatusAll= jobClient.getAllJobs();

    JobStatus jobStatus=null;

    if(jobStatusAll==null||jobStatusAll.length<=0){//当前集群中没有job,集群重启或者原本就没有job都可能造成这样的结果

           ClusterStatus clusterStatus=jobClient.getClusterStatus(true);

       System.out.println("集群"+clusterStatus+"当天到目前为止还没有运行过job!");

    }else{

       System.out.println("job的总数是："+jobStatusAll.length);

       for(int i=0;i<jobStatusAll.length;i++){//输出job列表中所有job的信息

          jobStatus=jobStatusAll[i];

          String jobName=jobClient.getJob(jobStatus.getJobID()).getJobName();

          int JobState=jobStatus.getRunState();

          String state="";

      

          switch (JobState) {

                 case 1:

                         state = "RUNNING";

                    break;

                      case 2:

                    state = "SUCCEEDED";

                    break;

                 case 3:

                    state = "FAILED";

                         break;

                 case 4:

                    state = "PREP";

                    break;

                 case 5:

                    state = "KILLED";

                    break;

                 default:

                    break;

               };

             

           System.out.println("job Id:"+jobStatus.getJobID());

           System.out.println("job Name:"+jobName);

           System.out.println("job User:"+jobStatus.getUsername());

           System.out.println("job Map-progress : " + jobStatus.mapProgress());

           System.out.println("job Map-total : "+jobClient.getMapTaskReports(jobStatus.getJobID()).length);

           System.out.println("job Reduce-progress : " + jobStatus.reduceProgress());

           System.out.println("job Reduce-total : "+jobClient.getReduceTaskReports(jobStatus.getJobID()).length);

           System.out.println("job Cleanupp-rogress : " + jobStatus.cleanupProgress());

       System.out.println("job Setup-progress : " + jobStatus.setupProgress());

                    System.out.println("job Priority : " + jobStatus.getJobPriority());

           System.out.println("job Counters " + jobClient.getJob(jobStatus.getJobID()).getCounters());             

                    System.out.println("job Start-time:"+jobStatus.getStartTime());

           

           URL url = new URL(jobClient.getJob(jobStatus.getJobID()).getTrackingURL()); //通过jobtracker网页连接获取其他信息 

                   BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()));  

                   String w = "";  

                   StringBuffer sb = new StringBuffer("");  

                   while ((w = br.readLine()) != null) {  

                       sb.append(w);  

                   }  

                   br.close(); //上边一小段是获取jobtracker网页信息

                   

                   Pattern p = Pattern.compile("Finished at:</b>.*?<br>");//过滤获取的信息，得到job所有时间

               java.util.regex.Matcher rm = p.matcher(sb);

               while (rm.find()) {

                  String tmp = rm.group();      

                  String str = tmp.substring(17, tmp.length()-4);

                  Date date = parse(str, "EEE MMM dd HH:mm:ss zzz yyyy", Locale.US);

                  SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

                  String f = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(date);

                  Date date1 = format.parse(f);

                  System.out.println("job Finish-time:"+date1.getTime());//job结束时间，  因为api中 jobStatus.getFinishTime()函数找不到，所以自己写了一个。   

              }

           

                   Pattern p1 = Pattern.compile("Finished in:</b>.*?<br>");//过滤获取的信息，得到job所有时间

                   java.util.regex.Matcher rm1 = p1.matcher(sb);

                   while (rm1.find()) {

                      String tmp = rm1.group();

                      System.out.println("job Used-time:"+tmp.substring(17, tmp.length()-4));//总共耗时

                   }

                   System.out.println("job State:"+state);//状态

                   

                   System.out.println("Map task:");//map详情信息

                   TaskReport[] mtp = jobClient.getMapTaskReports(jobStatus.getJobID());

                    for(int j=0;j<mtp.length;j++){

                      System.out.println("task id:"+mtp[j].getTaskId());  

                      System.out.println("task Start-time:"+mtp[j].getStartTime()); 

                      System.out.println("task Finish-time:"+mtp[j].getFinishTime()); 

                      System.out.println("task Progress:"+mtp[j].getProgress()); 

                      System.out.println("task Status:"+mtp[j].getCurrentStatus()); 

                      System.out.println("task Counters:"+mtp[j].getCounters()); 

                    }

                    

                    System.out.println("Reduce task:");//reduce详情信息

                    TaskReport[] rtp = jobClient.getReduceTaskReports(jobStatus.getJobID());

                    for(int j=0;j<rtp.length;j++){

                     System.out.println("task id:"+rtp[j].getTaskId());  

                     System.out.println("task Start-time:"+rtp[j].getStartTime()); 

                     System.out.println("task Finish-time:"+rtp[j].getFinishTime()); 

                     System.out.println("task Progress:"+rtp[j].getProgress()); 

                     System.out.println("task Status:"+rtp[j].getCurrentStatus()); 

                     System.out.println("task Counters:"+rtp[j].getCounters()); 

                     }

                 

       }

    }  

    }

       

    

    public static Date parse(String str, String pattern, Locale locale) {//转为本地的时间格式

        if(str == null || pattern == null) {

            return null;

        }

        try {

            return new SimpleDateFormat(pattern, locale).parse(str);

        } catch (ParseException e) {

            e.printStackTrace();

        }

        return null;

    }

     

    public static Configuration getsConf(){//设置配置信息

     Configuration conf=new Configuration();

      conf.set("mapred.job.tracker", HOST+":"+JOBTRACKER_PORT);

      conf.set("fs.default.name", NAMENODE_PORT);

     return conf;

    }

public int run(String[] arg0) throws Exception {

// TODO Auto-generated method stub

return 0;

}

}

执行结果：

The name of task:

14/12/11 16:16:34 WARN conf.Configuration: fs.default.name is deprecated. Instead, use fs.defaultFS

14/12/11 16:16:34 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

job的总数是：1

job Id:job_201412111355_0001

job Name:word count

job User:root

job Map-progress : 1.0

job Map-total : 1

job Reduce-progress : 1.0

job Reduce-total : 1

job Cleanupp-rogress : 1.0

job Setup-progress : 1.0

job Priority : NORMAL

job Counters Counters: 32

File System Counters

FILE: Number of bytes read=739

FILE: Number of bytes written=343517

FILE: Number of read operations=0

FILE: Number of large read operations=0

FILE: Number of write operations=0

HDFS: Number of bytes read=1655

HDFS: Number of bytes written=485

HDFS: Number of read operations=2

HDFS: Number of large read operations=0

HDFS: Number of write operations=1

Job Counters 

Launched map tasks=1

Launched reduce tasks=1

Data-local map tasks=1

Total time spent by all maps in occupied slots (ms)=74651

Total time spent by all reduces in occupied slots (ms)=18656

Total time spent by all maps waiting after reserving slots (ms)=0

Total time spent by all reduces waiting after reserving slots (ms)=0

Map-Reduce Framework

Map input records=250

Map output records=250

Map output bytes=2316

Input split bytes=91

Combine input records=250

Combine output records=64

Reduce input groups=64

Reduce shuffle bytes=739

Reduce input records=64

Reduce output records=64

Spilled Records=128

CPU time spent (ms)=4170

Physical memory (bytes) snapshot=295612416

Virtual memory (bytes) snapshot=2769776640

Total committed heap usage (bytes)=131207168

job Start-time:1418285681671

job Finish-time:1418285792000

job Used-time:1mins, 51sec

job State:SUCCEEDED

Map task:

task id:task_201412111355_0001_m_000000

task Start-time:1418285732395

task Finish-time:1418285768225

task Progress:1.0

task Status:COMPLETE

task Counters:Counters: 21

File System Counters

FILE: Number of bytes read=0

FILE: Number of bytes written=171928

FILE: Number of read operations=0

FILE: Number of large read operations=0

FILE: Number of write operations=0

HDFS: Number of bytes read=1655

HDFS: Number of bytes written=0

HDFS: Number of read operations=2

HDFS: Number of large read operations=0

HDFS: Number of write operations=0

Map-Reduce Framework

Map input records=250

Map output records=250

Map output bytes=2316

Input split bytes=91

Combine input records=250

Combine output records=64

Spilled Records=64

CPU time spent (ms)=2000

Physical memory (bytes) snapshot=182071296

Virtual memory (bytes) snapshot=1379409920

Total committed heap usage (bytes)=115478528

Reduce task:

task id:task_201412111355_0001_r_000000

task Start-time:1418285768231

task Finish-time:1418285787457

task Progress:1.0

task Status:COMPLETE

task Counters:Counters: 21

File System Counters

FILE: Number of bytes read=739

FILE: Number of bytes written=171589

FILE: Number of read operations=0

FILE: Number of large read operations=0

FILE: Number of write operations=0

HDFS: Number of bytes read=0

HDFS: Number of bytes written=485

HDFS: Number of read operations=0

HDFS: Number of large read operations=0

HDFS: Number of write operations=1

Map-Reduce Framework

Combine input records=0

Combine output records=0

Reduce input groups=64

Reduce shuffle bytes=739

Reduce input records=64

Reduce output records=64

Spilled Records=64

CPU time spent (ms)=2170

Physical memory (bytes) snapshot=113541120

Virtual memory (bytes) snapshot=1390366720

Total committed heap usage (bytes)=15728640

这只是job信息的获取初期结果，还需要再将信息抽取，存储，显示等！