Examples with JobType com.flink.platform.common.enums.JobType used on opensource projects

Example 1 with JobType

use of com.flink.platform.common.enums.JobType in project flink-platform-backend by itinycheng.

the class ProcessJobService method processJob.

public JobRunInfo processJob(final long jobId, final long flowRunId) throws Exception {
    JobCommand jobCommand = null;
    JobInfo jobInfo = null;
    try {
        // step 1: get job info
        jobInfo = jobInfoService.getOne(new QueryWrapper<JobInfo>().lambda().eq(JobInfo::getId, jobId).eq(JobInfo::getStatus, JobStatus.ONLINE));
        if (jobInfo == null) {
            throw new JobCommandGenException(String.format("The job: %s is no longer exists or in delete status.", jobId));
        }
        // step 2: replace variables in the sql statement
        JobInfo finalJobInfo = jobInfo;
        Map<String, Object> variableMap = Arrays.stream(SqlVar.values()).filter(sqlVar -> sqlVar.type == SqlVar.VarType.VARIABLE).filter(sqlVar -> finalJobInfo.getSubject().contains(sqlVar.variable)).map(sqlVar -> Pair.of(sqlVar.variable, sqlVar.valueProvider.apply(finalJobInfo))).collect(toMap(Pair::getLeft, Pair::getRight));
        MapUtils.emptyIfNull(finalJobInfo.getVariables()).forEach((name, value) -> {
            SqlVar sqlVar = SqlVar.matchPrefix(name);
            variableMap.put(name, sqlVar.valueProvider.apply(value));
        });
        // replace variable with actual value
        for (Map.Entry<String, Object> entry : variableMap.entrySet()) {
            String originSubject = jobInfo.getSubject();
            String distSubject = originSubject.replace(entry.getKey(), entry.getValue().toString());
            jobInfo.setSubject(distSubject);
        }
        JobType jobType = jobInfo.getType();
        String version = jobInfo.getVersion();
        // step 3: build job command, create a SqlContext if needed
        jobCommand = jobCommandBuilders.stream().filter(builder -> builder.isSupported(jobType, version)).findFirst().orElseThrow(() -> new JobCommandGenException("No available job command builder")).buildCommand(jobInfo);
        // step 4: submit job
        LocalDateTime submitTime = LocalDateTime.now();
        String commandString = jobCommand.toCommandString();
        JobCallback callback = jobCommandExecutors.stream().filter(executor -> executor.isSupported(jobType)).findFirst().orElseThrow(() -> new JobCommandGenException("No available job command executor")).execCommand(commandString);
        // step 5: write job run info to db
        ExecutionStatus executionStatus = getExecutionStatus(jobType, callback);
        JobRunInfo jobRunInfo = new JobRunInfo();
        jobRunInfo.setName(jobInfo.getName() + "-" + System.currentTimeMillis());
        jobRunInfo.setJobId(jobInfo.getId());
        jobRunInfo.setFlowRunId(flowRunId);
        jobRunInfo.setDeployMode(jobInfo.getDeployMode());
        jobRunInfo.setExecMode(jobInfo.getExecMode());
        jobRunInfo.setSubject(jobInfo.getSubject());
        jobRunInfo.setStatus(executionStatus);
        jobRunInfo.setVariables(JsonUtil.toJsonString(variableMap));
        jobRunInfo.setBackInfo(JsonUtil.toJsonString(callback));
        jobRunInfo.setSubmitTime(submitTime);
        if (executionStatus.isTerminalState()) {
            jobRunInfo.setStopTime(LocalDateTime.now());
        }
        jobRunInfoService.save(jobRunInfo);
        // step 6: print job command info
        log.info("Job: {} submitted, time: {}", jobId, System.currentTimeMillis());
        return jobRunInfo;
    } finally {
        if (jobInfo != null && jobInfo.getType() == JobType.FLINK_SQL && jobCommand != null) {
            try {
                FlinkCommand flinkCommand = (FlinkCommand) jobCommand;
                if (flinkCommand.getMainArgs() != null) {
                    Files.deleteIfExists(Paths.get(flinkCommand.getMainArgs()));
                }
            } catch (Exception e) {
                log.warn("Delete sql context file failed", e);
            }
        }
    }
}