Examples with JobFlowRun com.flink.platform.dao.entity.JobFlowRun used on opensource projects

Example 1 with JobFlowRun

use of com.flink.platform.dao.entity.JobFlowRun in project flink-platform-backend by itinycheng.

the class FlowExecuteThread method completeAndNotify.

/**
 * Update status of jobFlow and send notification.
 */
private void completeAndNotify(JobFlowDag flow) {
    ExecutionStatus finalStatus = JobFlowDagHelper.getDagState(flow);
    if (finalStatus.isTerminalState()) {
        jobFlowRun.setStatus(finalStatus);
        JobFlowRun newJobFlowRun = new JobFlowRun();
        newJobFlowRun.setId(jobFlowRun.getId());
        newJobFlowRun.setStatus(finalStatus);
        jobFlowRunService.updateById(newJobFlowRun);
        // send notification.
        sendNotification();
    }
}

Example 2 with JobFlowRun

use of com.flink.platform.dao.entity.JobFlowRun in project flink-platform-backend by itinycheng.

the class InitJobFlowScheduler method appendExistedJobFlowRunToScheduler.

public void appendExistedJobFlowRunToScheduler() {
    List<JobFlowRun> unfinishedFlowRunList = jobFlowRunService.list(new QueryWrapper<JobFlowRun>().lambda().eq(JobFlowRun::getHost, Constant.HOST_IP).in(JobFlowRun::getStatus, getNonTerminals()));
    for (JobFlowRun jobFlowRun : unfinishedFlowRunList) {
        DAG<Long, JobVertex, JobEdge> flow = jobFlowRun.getFlow();
        // Update status of JobVertex in flow.
        jobRunInfoService.list(new QueryWrapper<JobRunInfo>().lambda().eq(JobRunInfo::getFlowRunId, jobFlowRun.getId())).forEach(jobRunInfo -> {
            JobVertex vertex = flow.getVertex(jobRunInfo.getJobId());
            vertex.setJobRunId(jobRunInfo.getId());
            vertex.setJobRunStatus(jobRunInfo.getStatus());
        });
        jobFlowScheduleService.registerToScheduler(jobFlowRun);
    }
}

Example 3 with JobFlowRun

use of com.flink.platform.dao.entity.JobFlowRun in project flink-platform-backend by itinycheng.

the class JobFlowRunner method execute.

@Override
public void execute(JobExecutionContext context) {
    JobDetail detail = context.getJobDetail();
    JobKey key = detail.getKey();
    String code = key.getName();
    synchronized (getProcessLock(code)) {
        // Get job flow info.
        JobFlow jobFlow = jobFlowService.getOne(new QueryWrapper<JobFlow>().lambda().eq(JobFlow::getCode, code).in(JobFlow::getStatus, ONLINE, SCHEDULING));
        if (jobFlow == null) {
            log.warn("The job flow: {} isn't exists or not in scheduling status", code);
            return;
        }
        // Validate flow json.
        DAG<Long, JobVertex, JobEdge> flow = jobFlow.getFlow();
        if (flow == null || flow.getVertices().isEmpty()) {
            log.warn("The job flow: {} doesn't contain any vertices", jobFlow.getCode());
            return;
        }
        // Avoid preforming the same job flow multiple times at the same time.
        JobFlowRun jobFlowRun = jobFlowRunService.getOne(new QueryWrapper<JobFlowRun>().lambda().eq(JobFlowRun::getFlowId, jobFlow.getId()).in(JobFlowRun::getStatus, getNonTerminals()));
        if (jobFlowRun != null) {
            log.warn("The job flow:{} is in non-terminal status, run id: {}", jobFlow.getId(), jobFlowRun.getId());
            return;
        }
        // Create job flow run instance.
        jobFlowRun = new JobFlowRun();
        jobFlowRun.setFlowId(jobFlow.getId());
        jobFlowRun.setName(String.join("-", jobFlow.getName(), jobFlow.getCode(), String.valueOf(System.currentTimeMillis())));
        jobFlowRun.setFlow(jobFlow.getFlow());
        jobFlowRun.setUserId(jobFlow.getUserId());
        jobFlowRun.setHost(Constant.HOST_IP);
        jobFlowRun.setPriority(jobFlow.getPriority());
        jobFlowRun.setAlerts(jobFlow.getAlerts());
        jobFlowRun.setStatus(SUBMITTED);
        jobFlowRunService.save(jobFlowRun);
        // register job flow run.
        jobFlowScheduleService.registerToScheduler(jobFlowRun);
        log.info("Job flow run: {} is created, job flow: {}, time: {}", jobFlowRun.getFlowId(), code, System.currentTimeMillis());
    }
}

Example 4 with JobFlowRun

use of com.flink.platform.dao.entity.JobFlowRun in project flink-platform-backend by itinycheng.

the class JobFlowScheduleService method registerToScheduler.

public synchronized void registerToScheduler(JobFlowRun jobFlowRun) {
    JobFlowDag flow = jobFlowRun.getFlow();
    if (flow == null || CollectionUtils.isEmpty(flow.getVertices())) {
        log.warn("No JobVertex found, no scheduling required, flow run id: {}", jobFlowRun.getId());
        return;
    }
    if (inFlightFlows.stream().anyMatch(inQueue -> inQueue.getId().equals(jobFlowRun.getId()))) {
        log.warn("The JobFlowRun already registered, jobFlowRun: {} ", jobFlowRun);
        return;
    }
    if (inFlightFlows.size() > 2 * workerConfig.getFlowExecThreads()) {
        log.warn("Not have enough resources to execute flow: {}", jobFlowRun);
        JobFlowRun newJobFlowRun = new JobFlowRun();
        newJobFlowRun.setId(jobFlowRun.getId());
        newJobFlowRun.setStatus(FAILURE);
        newJobFlowRun.setEndTime(LocalDateTime.now());
        jobFlowRunService.updateById(newJobFlowRun);
        return;
    }
    inFlightFlows.offer(jobFlowRun);
}

Example 5 with JobFlowRun

use of com.flink.platform.dao.entity.JobFlowRun in project flink-platform-backend by itinycheng.

the class JobFlowController method runOnce.

@GetMapping(value = "/schedule/runOnce/{flowId}")
public ResultInfo<Long> runOnce(@PathVariable Long flowId) {
    JobFlow jobFlow = jobFlowService.getById(flowId);
    JobFlowStatus status = jobFlow.getStatus();
    if (status == null || !status.isRunnable()) {
        return failure(NOT_RUNNABLE_STATUS);
    }
    // TODO better in sync lock.
    JobFlowRun jobFlowRun = jobFlowRunService.getById(flowId);
    if (jobFlowRun != null && !jobFlowRun.getStatus().isTerminalState()) {
        return failure(SERVICE_ERROR);
    }
    JobFlowQuartzInfo jobFlowQuartzInfo = new JobFlowQuartzInfo(jobFlow);
    if (quartzService.runOnce(jobFlowQuartzInfo)) {
        return ResultInfo.success(flowId);
    } else {
        return failure(SERVICE_ERROR);
    }
}