flink启动job_Flink源码第一篇：Flink之Job启动流程

Flink启动Job

$ ./bin/flink run examples/streaming/SocketWindowWordCount.jar

先看一下flink脚本内部调度流程

cat flink

大家可以看到flink内部执行流程

• 初始化config

• 调用CliFrontend

CliFrontend

类目录：package org.apache.flink.client.cli;打开这个类大家可以看到熟悉的命令

然后观察main函数：

parseParameters执行了什么？

run(params)内部时如何调用的？最后调用了runProgram(customCommandLine, commandLine, runOptions, program);

runProgram()执行流程

private <T> void runProgram(

CustomCommandLine<T> customCommandLine,

CommandLine commandLine,

RunOptions runOptions,

PackagedProgram program) throws ProgramInvocationException, FlinkException {

final ClusterDescriptor<T> clusterDescriptor = customCommandLine.createClusterDescriptor(commandLine);

try {

//如果命令中制定了clusterid，则会返回，否则返回空

final T clusterId = customCommandLine.getClusterId(commandLine);

//初始化客户端

final ClusterClient<T> client;

// directly deploy the job if the cluster is started in job mode and detached

// 如果集群以作业模式启动并分离，则直接部署作业

if (clusterId == null && runOptions.getDetachedMode()) {

int parallelism = runOptions.getParallelism() == -1 ? defaultParallelism : runOptions.getParallelism();

//初始化jobGraph

final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(program, configuration, parallelism);

final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);

//开始执行jobGraph

client = clusterDescriptor.deployJobCluster(

clusterSpecification,

jobGraph,

runOptions.getDetachedMode());

//job已经提交，并且jobid打印出来

logAndSysout("Job has been submitted with JobID " + jobGraph.getJobID());

try {

client.shutdown();

} catch (Exception e) {

LOG.info("Could not properly shut down the client.", e);

}

} else {

final Thread shutdownHook;

if (clusterId != null) {

client = clusterDescriptor.retrieve(clusterId);

shutdownHook = null;

} else {

// also in job mode we have to deploy a session cluster because the job

// might consist of multiple parts (e.g. when using collect)

final ClusterSpecification clusterSpecification = customCommandLine.getClusterSpecification(commandLine);

client = clusterDescriptor.deploySessionCluster(clusterSpecification);

// if not running in detached mode, add a shutdown hook to shut down cluster if client exits

// there's a race-condition here if cli is killed before shutdown hook is installed

if (!runOptions.getDetachedMode() && runOptions.isShutdownOnAttachedExit()) {

shutdownHook = ShutdownHookUtil.addShutdownHook(client::shutDownCluster, client.getClass().getSimpleName(), LOG);

} else {

shutdownHook = null;

}

}

try {

client.setPrintStatusDuringExecution(runOptions.getStdoutLogging());

client.setDetached(runOptions.getDetachedMode());

LOG.debug("{}", runOptions.getSavepointRestoreSettings());

int userParallelism = runOptions.getParallelism();

LOG.debug("User parallelism is set to {}", userParallelism);

if (ExecutionConfig.PARALLELISM_DEFAULT == userParallelism) {

userParallelism = defaultParallelism;

}

executeProgram(program, client, userParallelism);

} finally {

if (clusterId == null && !client.isDetached()) {

// terminate the cluster only if we have started it before and if it's not detached

try {

client.shutDownCluster();

} catch (final Exception e) {

LOG.info("Could not properly terminate the Flink cluster.", e);

}

if (shutdownHook != null) {

// we do not need the hook anymore as we have just tried to shutdown the cluster.

ShutdownHookUtil.removeShutdownHook(shutdownHook, client.getClass().getSimpleName(), LOG);

}

}

try {

client.shutdown();

} catch (Exception e) {

LOG.info("Could not properly shut down the client.", e);

}

}

}

} finally {

try {

clusterDescriptor.close();

} catch (Exception e) {

LOG.info("Could not properly close the cluster descriptor.", e);

}

}

}