Examples with JobVertex com.flink.platform.common.model.JobVertex used on opensource projects

Example 1 with JobVertex

use of com.flink.platform.common.model.JobVertex in project flink-platform-backend by itinycheng.

the class InitJobFlowScheduler method appendExistedJobFlowRunToScheduler.

public void appendExistedJobFlowRunToScheduler() {
    List<JobFlowRun> unfinishedFlowRunList = jobFlowRunService.list(new QueryWrapper<JobFlowRun>().lambda().eq(JobFlowRun::getHost, Constant.HOST_IP).in(JobFlowRun::getStatus, getNonTerminals()));
    for (JobFlowRun jobFlowRun : unfinishedFlowRunList) {
        DAG<Long, JobVertex, JobEdge> flow = jobFlowRun.getFlow();
        // Update status of JobVertex in flow.
        jobRunInfoService.list(new QueryWrapper<JobRunInfo>().lambda().eq(JobRunInfo::getFlowRunId, jobFlowRun.getId())).forEach(jobRunInfo -> {
            JobVertex vertex = flow.getVertex(jobRunInfo.getJobId());
            vertex.setJobRunId(jobRunInfo.getId());
            vertex.setJobRunStatus(jobRunInfo.getStatus());
        });
        jobFlowScheduleService.registerToScheduler(jobFlowRun);
    }
}

Example 2 with JobVertex

use of com.flink.platform.common.model.JobVertex in project flink-platform-backend by itinycheng.

the class JobFlowRunner method execute.

@Override
public void execute(JobExecutionContext context) {
    JobDetail detail = context.getJobDetail();
    JobKey key = detail.getKey();
    String code = key.getName();
    synchronized (getProcessLock(code)) {
        // Get job flow info.
        JobFlow jobFlow = jobFlowService.getOne(new QueryWrapper<JobFlow>().lambda().eq(JobFlow::getCode, code).in(JobFlow::getStatus, ONLINE, SCHEDULING));
        if (jobFlow == null) {
            log.warn("The job flow: {} isn't exists or not in scheduling status", code);
            return;
        }
        // Validate flow json.
        DAG<Long, JobVertex, JobEdge> flow = jobFlow.getFlow();
        if (flow == null || flow.getVertices().isEmpty()) {
            log.warn("The job flow: {} doesn't contain any vertices", jobFlow.getCode());
            return;
        }
        // Avoid preforming the same job flow multiple times at the same time.
        JobFlowRun jobFlowRun = jobFlowRunService.getOne(new QueryWrapper<JobFlowRun>().lambda().eq(JobFlowRun::getFlowId, jobFlow.getId()).in(JobFlowRun::getStatus, getNonTerminals()));
        if (jobFlowRun != null) {
            log.warn("The job flow:{} is in non-terminal status, run id: {}", jobFlow.getId(), jobFlowRun.getId());
            return;
        }
        // Create job flow run instance.
        jobFlowRun = new JobFlowRun();
        jobFlowRun.setFlowId(jobFlow.getId());
        jobFlowRun.setName(String.join("-", jobFlow.getName(), jobFlow.getCode(), String.valueOf(System.currentTimeMillis())));
        jobFlowRun.setFlow(jobFlow.getFlow());
        jobFlowRun.setUserId(jobFlow.getUserId());
        jobFlowRun.setHost(Constant.HOST_IP);
        jobFlowRun.setPriority(jobFlow.getPriority());
        jobFlowRun.setAlerts(jobFlow.getAlerts());
        jobFlowRun.setStatus(SUBMITTED);
        jobFlowRunService.save(jobFlowRun);
        // register job flow run.
        jobFlowScheduleService.registerToScheduler(jobFlowRun);
        log.info("Job flow run: {} is created, job flow: {}, time: {}", jobFlowRun.getFlowId(), code, System.currentTimeMillis());
    }
}

Example 3 with JobVertex

use of com.flink.platform.common.model.JobVertex in project flink-platform-backend by itinycheng.

the class JobFlowDagHelper method isPreconditionSatisfied.

public static boolean isPreconditionSatisfied(JobVertex toVertex, DAG<Long, JobVertex, JobEdge> dag) {
    Collection<JobVertex> preVertices = dag.getPreVertices(toVertex);
    if (CollectionUtils.isEmpty(preVertices)) {
        return true;
    }
    ExecutionCondition precondition = toVertex.getPrecondition();
    if (precondition == AND) {
        return preVertices.stream().allMatch(fromVertex -> fromVertex.getJobRunStatus() == dag.getEdge(fromVertex, toVertex).getExpectStatus());
    } else if (precondition == OR) {
        return preVertices.stream().anyMatch(fromVertex -> fromVertex.getJobRunStatus() == dag.getEdge(fromVertex, toVertex).getExpectStatus());
    } else {
        throw new IllegalStateException("Can't handle precondition status: " + precondition);
    }
}

Example 4 with JobVertex

use of com.flink.platform.common.model.JobVertex in project flink-platform-backend by itinycheng.

the class JobFlowDagHelper method getDagState.

// TODO : dag can not have executable vertices
@Nonnull
public static ExecutionStatus getDagState(DAG<Long, JobVertex, JobEdge> dag) {
    Set<ExecutionStatus> vertexStatusList = dag.getVertices().stream().map(JobVertex::getJobRunStatus).filter(Objects::nonNull).collect(toSet());
    ExecutionStatus status;
    if (vertexStatusList.contains(ERROR)) {
        status = ERROR;
    } else if (vertexStatusList.contains(NOT_EXIST)) {
        status = NOT_EXIST;
    } else if (vertexStatusList.contains(FAILURE)) {
        status = FAILURE;
    } else if (vertexStatusList.contains(ABNORMAL)) {
        status = ABNORMAL;
    } else if (vertexStatusList.contains(KILLED)) {
        status = KILLED;
    } else if (vertexStatusList.contains(RUNNING)) {
        status = RUNNING;
    } else if (vertexStatusList.contains(SUCCESS)) {
        status = SUCCESS;
    } else {
        status = SUBMITTED;
    }
    return status;
}

Example 5 with JobVertex

use of com.flink.platform.common.model.JobVertex in project flink-platform-backend by itinycheng.

the class JobFlowDagHelper method getNextExecutableVertices.

private static Set<JobVertex> getNextExecutableVertices(Collection<JobVertex> fromVertices, DAG<Long, JobVertex, JobEdge> dag) {
    // Get the edges whose status matched his formVertex's status.
    Set<JobEdge> statusMatchedEdgeSet = fromVertices.stream().flatMap(fromVertex -> dag.getEdgesFromVertex(fromVertex).stream().map(edge -> edge.unwrap(JobEdge.class)).filter(edge -> edge.getExpectStatus() == fromVertex.getJobRunStatus())).collect(toSet());
    // Get the executable vertices.
    Set<JobVertex> executableToVertices = statusMatchedEdgeSet.stream().map(edge -> dag.getVertex(edge.getToVId())).filter(toVertex -> isPreconditionSatisfied(toVertex, dag)).collect(toSet());
    // If toVertex is executed, use it as fromVertex to find the next executable vertex.
    Set<JobVertex> executedVertices = new HashSet<>();
    Set<JobVertex> unExecutedVertices = new HashSet<>();
    for (JobVertex executableToVertex : executableToVertices) {
        if (executableToVertex.getJobRunStatus() != null) {
            executedVertices.add(executableToVertex);
        } else {
            unExecutedVertices.add(executableToVertex);
        }
    }
    if (CollectionUtils.isNotEmpty(executedVertices)) {
        unExecutedVertices.addAll(getNextExecutableVertices(executedVertices, dag));
    }
    return unExecutedVertices;
}