Examples with JobFlowDag com.flink.platform.dao.entity.JobFlowDag used on opensource projects

Example 1 with JobFlowDag

use of com.flink.platform.dao.entity.JobFlowDag in project flink-platform-backend by itinycheng.

the class JobFlowController method start.

@GetMapping(value = "/schedule/start/{flowId}")
public ResultInfo<Long> start(@PathVariable Long flowId) {
    JobFlowRequest jobFlowRequest = new JobFlowRequest();
    jobFlowRequest.setId(flowId);
    String errorMsg = jobFlowRequest.verifyId();
    if (StringUtils.isNotBlank(errorMsg)) {
        return failure(ERROR_PARAMETER, errorMsg);
    }
    JobFlow jobFlow = jobFlowService.getById(jobFlowRequest.getId());
    JobFlowStatus status = jobFlow.getStatus();
    if (status == null || !status.isRunnable()) {
        return failure(NOT_RUNNABLE_STATUS);
    }
    if (StringUtils.isEmpty(jobFlow.getCronExpr())) {
        return failure(NO_CRONTAB_SET);
    }
    JobFlowDag flow = jobFlow.getFlow();
    if (flow == null || jobFlowService.containsStreamingJob(flow)) {
        return failure(UNABLE_SCHEDULE_STREAMING_JOB);
    }
    jobFlowQuartzService.scheduleJob(jobFlow);
    return ResultInfo.success(flowId);
}

Example 2 with JobFlowDag

use of com.flink.platform.dao.entity.JobFlowDag in project flink-platform-backend by itinycheng.

the class JobFlowScheduleService method registerToScheduler.

public synchronized void registerToScheduler(JobFlowRun jobFlowRun) {
    JobFlowDag flow = jobFlowRun.getFlow();
    if (flow == null || CollectionUtils.isEmpty(flow.getVertices())) {
        log.warn("No JobVertex found, no scheduling required, flow run id: {}", jobFlowRun.getId());
        return;
    }
    if (inFlightFlows.stream().anyMatch(inQueue -> inQueue.getId().equals(jobFlowRun.getId()))) {
        log.warn("The JobFlowRun already registered, jobFlowRun: {} ", jobFlowRun);
        return;
    }
    if (inFlightFlows.size() > 2 * workerConfig.getFlowExecThreads()) {
        log.warn("Not have enough resources to execute flow: {}", jobFlowRun);
        JobFlowRun newJobFlowRun = new JobFlowRun();
        newJobFlowRun.setId(jobFlowRun.getId());
        newJobFlowRun.setStatus(FAILURE);
        newJobFlowRun.setEndTime(LocalDateTime.now());
        jobFlowRunService.updateById(newJobFlowRun);
        return;
    }
    inFlightFlows.offer(jobFlowRun);
}

Example 3 with JobFlowDag

use of com.flink.platform.dao.entity.JobFlowDag in project flink-platform-backend by itinycheng.

the class JobFlowService method updateFlowById.

@Transactional(rollbackFor = Exception.class)
public void updateFlowById(JobFlow origin) {
    if (origin.getId() == null) {
        return;
    }
    JobFlow newJobFlow = new JobFlow();
    newJobFlow.setId(origin.getId());
    if (origin.getFlow() != null) {
        newJobFlow.setFlow(origin.getFlow());
    } else {
        newJobFlow.setFlow(new JobFlowDag());
    }
    updateById(newJobFlow);
}

Example 4 with JobFlowDag

use of com.flink.platform.dao.entity.JobFlowDag in project flink-platform-backend by itinycheng.

the class FlowExecuteThread method run.

@Override
public void run() {
    // Update status of jobFlowRun.
    JobFlowRun newJobFlowRun = new JobFlowRun();
    newJobFlowRun.setId(jobFlowRun.getId());
    newJobFlowRun.setStatus(RUNNING);
    jobFlowRunService.updateById(newJobFlowRun);
    // Process job flow.
    JobFlowDag flow = jobFlowRun.getFlow();
    flow.getBeginVertices().forEach(jobVertex -> execVertex(jobVertex, flow));
    // Wait until all vertices are executed.
    while (JobFlowDagHelper.hasUnExecutedVertices(flow)) {
        ThreadUtil.sleep(5000);
    }
    // Wait for all jobs complete.
    CompletableFuture.allOf(runningJobs.values().toArray(new CompletableFuture[0])).thenAccept(unused -> completeAndNotify(flow)).thenAccept(unused -> jobExecService.shutdownNow());
}

Example 5 with JobFlowDag

use of com.flink.platform.dao.entity.JobFlowDag in project flink-platform-backend by itinycheng.

the class JobFlowTest method test1.

@Test
public void test1() {
    JobFlowDag dag = new JobFlowDag();
    JobVertex jobVertex1 = new JobVertex(19L, 19L);
    JobVertex jobVertex2 = new JobVertex(20L, 20L);
    dag.addVertex(jobVertex1);
    dag.addVertex(jobVertex2);
    JobEdge jobEdge = new JobEdge(19L, 20L, SUCCESS);
    dag.addEdge(jobEdge);
    JobFlowRequest jobFlowRequest = new JobFlowRequest();
    jobFlowRequest.setCode(UuidGenerator.generateShortUuid());
    jobFlowRequest.setName("test_1");
    jobFlowRequest.setUserId(0L);
    jobFlowRequest.setDescription("description");
    jobFlowRequest.setCronExpr("0 0/10 * * * ?");
    jobFlowRequest.setFlow(dag);
    jobFlowRequest.setPriority(8);
    jobFlowRequest.setAlerts(new LongArrayList());
    jobFlowRequest.setStatus(OFFLINE);
    String json = JsonUtil.toJsonString(jobFlowRequest.getJobFlow());
    System.out.println(json);
}