Examples with ExecutionStatus com.flink.platform.common.enums.ExecutionStatus used on opensource projects

Example 1 with ExecutionStatus

use of com.flink.platform.common.enums.ExecutionStatus in project flink-platform-backend by itinycheng.

the class FlowExecuteThread method completeAndNotify.

/**
 * Update status of jobFlow and send notification.
 */
private void completeAndNotify(JobFlowDag flow) {
    ExecutionStatus finalStatus = JobFlowDagHelper.getDagState(flow);
    if (finalStatus.isTerminalState()) {
        jobFlowRun.setStatus(finalStatus);
        JobFlowRun newJobFlowRun = new JobFlowRun();
        newJobFlowRun.setId(jobFlowRun.getId());
        newJobFlowRun.setStatus(finalStatus);
        jobFlowRunService.updateById(newJobFlowRun);
        // send notification.
        sendNotification();
    }
}

Example 2 with ExecutionStatus

use of com.flink.platform.common.enums.ExecutionStatus in project flink-platform-backend by itinycheng.

the class JobExecuteThread method call.

@Override
public JobResponse call() {
    Long jobId = jobVertex.getJobId();
    Long jobRunId = jobVertex.getJobRunId();
    try {
        // Step 1: get job info
        JobInfo jobInfo = jobInfoService.getOne(new QueryWrapper<JobInfo>().lambda().eq(JobInfo::getId, jobId).eq(JobInfo::getStatus, JobStatus.ONLINE));
        if (jobInfo == null) {
            log.warn("The job:{} is no longer exists or not in ready/scheduled status.", jobId);
            return new JobResponse(jobId, jobRunId, NOT_EXIST);
        }
        // Step 2: build route url, set localhost as default url if not specified.
        String routeUrl = jobInfo.getRouteUrl();
        routeUrl = HttpUtil.getUrlOrDefault(routeUrl);
        // Step 3: process job and get jobRun.
        JobRunInfo jobRunInfo;
        if (jobRunId != null) {
            jobRunInfo = jobRunInfoService.getById(jobRunId);
            log.info("Job:{} already submitted, runId = {}.", jobId, jobRunId);
        } else {
            jobRunInfo = processRemoteJob(routeUrl, jobId);
        }
        if (jobRunInfo == null) {
            log.warn("The jobRun:{} is no longer exists.", jobRunId);
            return new JobResponse(jobId, jobRunId, NOT_EXIST);
        }
        // Step 4: Update jobRunId in Memory.
        jobRunId = jobRunInfo.getId();
        // Step 5: Wait for job complete and get final status.
        ExecutionStatus status = jobRunInfo.getStatus();
        if (status == null || !status.isTerminalState()) {
            StatusInfo statusInfo = waitForComplete(routeUrl, jobRunInfo);
            if (statusInfo != null) {
                status = statusInfo.getStatus();
                updateJobRunInfo(jobRunId, statusInfo.getStatus(), statusInfo.getEndTime());
            }
        }
        return new JobResponse(jobId, jobRunId, status);
    } catch (Exception e) {
        log.error("Submit job and wait for complete failed.", e);
        updateJobRunInfo(jobRunId, ERROR, LocalDateTime.now());
        return new JobResponse(jobId, jobRunId, ERROR);
    }
}

Example 3 with ExecutionStatus

use of com.flink.platform.common.enums.ExecutionStatus in project flink-platform-backend by itinycheng.

the class ProcessJobService method processJob.

public JobRunInfo processJob(final long jobId, final long flowRunId) throws Exception {
    JobCommand jobCommand = null;
    JobInfo jobInfo = null;
    try {
        // step 1: get job info
        jobInfo = jobInfoService.getOne(new QueryWrapper<JobInfo>().lambda().eq(JobInfo::getId, jobId).eq(JobInfo::getStatus, JobStatus.ONLINE));
        if (jobInfo == null) {
            throw new JobCommandGenException(String.format("The job: %s is no longer exists or in delete status.", jobId));
        }
        // step 2: replace variables in the sql statement
        JobInfo finalJobInfo = jobInfo;
        Map<String, Object> variableMap = Arrays.stream(SqlVar.values()).filter(sqlVar -> sqlVar.type == SqlVar.VarType.VARIABLE).filter(sqlVar -> finalJobInfo.getSubject().contains(sqlVar.variable)).map(sqlVar -> Pair.of(sqlVar.variable, sqlVar.valueProvider.apply(finalJobInfo))).collect(toMap(Pair::getLeft, Pair::getRight));
        MapUtils.emptyIfNull(finalJobInfo.getVariables()).forEach((name, value) -> {
            SqlVar sqlVar = SqlVar.matchPrefix(name);
            variableMap.put(name, sqlVar.valueProvider.apply(value));
        });
        // replace variable with actual value
        for (Map.Entry<String, Object> entry : variableMap.entrySet()) {
            String originSubject = jobInfo.getSubject();
            String distSubject = originSubject.replace(entry.getKey(), entry.getValue().toString());
            jobInfo.setSubject(distSubject);
        }
        JobType jobType = jobInfo.getType();
        String version = jobInfo.getVersion();
        // step 3: build job command, create a SqlContext if needed
        jobCommand = jobCommandBuilders.stream().filter(builder -> builder.isSupported(jobType, version)).findFirst().orElseThrow(() -> new JobCommandGenException("No available job command builder")).buildCommand(jobInfo);
        // step 4: submit job
        LocalDateTime submitTime = LocalDateTime.now();
        String commandString = jobCommand.toCommandString();
        JobCallback callback = jobCommandExecutors.stream().filter(executor -> executor.isSupported(jobType)).findFirst().orElseThrow(() -> new JobCommandGenException("No available job command executor")).execCommand(commandString);
        // step 5: write job run info to db
        ExecutionStatus executionStatus = getExecutionStatus(jobType, callback);
        JobRunInfo jobRunInfo = new JobRunInfo();
        jobRunInfo.setName(jobInfo.getName() + "-" + System.currentTimeMillis());
        jobRunInfo.setJobId(jobInfo.getId());
        jobRunInfo.setFlowRunId(flowRunId);
        jobRunInfo.setDeployMode(jobInfo.getDeployMode());
        jobRunInfo.setExecMode(jobInfo.getExecMode());
        jobRunInfo.setSubject(jobInfo.getSubject());
        jobRunInfo.setStatus(executionStatus);
        jobRunInfo.setVariables(JsonUtil.toJsonString(variableMap));
        jobRunInfo.setBackInfo(JsonUtil.toJsonString(callback));
        jobRunInfo.setSubmitTime(submitTime);
        if (executionStatus.isTerminalState()) {
            jobRunInfo.setStopTime(LocalDateTime.now());
        }
        jobRunInfoService.save(jobRunInfo);
        // step 6: print job command info
        log.info("Job: {} submitted, time: {}", jobId, System.currentTimeMillis());
        return jobRunInfo;
    } finally {
        if (jobInfo != null && jobInfo.getType() == JobType.FLINK_SQL && jobCommand != null) {
            try {
                FlinkCommand flinkCommand = (FlinkCommand) jobCommand;
                if (flinkCommand.getMainArgs() != null) {
                    Files.deleteIfExists(Paths.get(flinkCommand.getMainArgs()));
                }
            } catch (Exception e) {
                log.warn("Delete sql context file failed", e);
            }
        }
    }
}

Example 4 with ExecutionStatus

use of com.flink.platform.common.enums.ExecutionStatus in project flink-platform-backend by itinycheng.

the class JobFlowDagHelper method getDagState.

// TODO : dag can not have executable vertices
@Nonnull
public static ExecutionStatus getDagState(DAG<Long, JobVertex, JobEdge> dag) {
    Set<ExecutionStatus> vertexStatusList = dag.getVertices().stream().map(JobVertex::getJobRunStatus).filter(Objects::nonNull).collect(toSet());
    ExecutionStatus status;
    if (vertexStatusList.contains(ERROR)) {
        status = ERROR;
    } else if (vertexStatusList.contains(NOT_EXIST)) {
        status = NOT_EXIST;
    } else if (vertexStatusList.contains(FAILURE)) {
        status = FAILURE;
    } else if (vertexStatusList.contains(ABNORMAL)) {
        status = ABNORMAL;
    } else if (vertexStatusList.contains(KILLED)) {
        status = KILLED;
    } else if (vertexStatusList.contains(RUNNING)) {
        status = RUNNING;
    } else if (vertexStatusList.contains(SUCCESS)) {
        status = SUCCESS;
    } else {
        status = SUBMITTED;
    }
    return status;
}

Example 5 with ExecutionStatus

use of com.flink.platform.common.enums.ExecutionStatus in project flink-platform-backend by itinycheng.

the class FlowExecuteThread method handleResponse.

private void handleResponse(JobResponse jobResponse, JobVertex jobVertex, JobFlowDag flow) {
    if (!isRunning) {
        return;
    }
    jobVertex.setJobRunId(jobResponse.getJobRunId());
    jobVertex.setJobRunStatus(jobResponse.getStatus());
    ExecutionStatus finalStatus = jobResponse.getStatus();
    if (ExecutionStatus.isStopFlowState(finalStatus)) {
        killFlow();
        return;
    }
    for (JobVertex nextVertex : flow.getNextVertices(jobVertex)) {
        if (JobFlowDagHelper.isPreconditionSatisfied(nextVertex, flow)) {
            execVertex(nextVertex, flow);
        }
    }
}