Examples with ParamAction com.dtstack.taier.pluginapi.pojo.ParamAction used on opensource projects

Example 11 with ParamAction

use of com.dtstack.taier.pluginapi.pojo.ParamAction in project Taier by DTStack.

the class ConsoleService method jobStick.

public Boolean jobStick(String jobId) {
    try {
        ScheduleEngineJobCache engineJobCache = scheduleEngineJobCacheMapper.getOne(jobId);
        if (null == engineJobCache) {
            return false;
        }
        // 只支持DB、PRIORITY两种调整顺序
        if (EJobCacheStage.DB.getStage() == engineJobCache.getStage() || EJobCacheStage.PRIORITY.getStage() == engineJobCache.getStage()) {
            ParamAction paramAction = PublicUtil.jsonStrToObject(engineJobCache.getJobInfo(), ParamAction.class);
            JobClient jobClient = new JobClient(paramAction);
            jobClient.setCallBack((jobStatus) -> {
                jobDealer.updateJobStatus(jobClient.getJobId(), jobStatus);
            });
            Long minPriority = scheduleEngineJobCacheMapper.minPriorityByStage(engineJobCache.getJobResource(), Lists.newArrayList(EJobCacheStage.PRIORITY.getStage()), engineJobCache.getNodeAddress());
            minPriority = minPriority == null ? 0 : minPriority;
            jobClient.setPriority(minPriority - 1);
            if (EJobCacheStage.PRIORITY.getStage() == engineJobCache.getStage()) {
                // 先将队列中的元素移除，重复插入会被忽略
                GroupPriorityQueue groupPriorityQueue = jobDealer.getGroupPriorityQueue(engineJobCache.getJobResource());
                groupPriorityQueue.remove(jobClient);
            }
            return jobDealer.addGroupPriorityQueue(engineJobCache.getJobResource(), jobClient, false, false);
        }
    } catch (Exception e) {
        LOGGER.error("jobStick error:", e);
    }
    return false;
}

Example 12 with ParamAction

use of com.dtstack.taier.pluginapi.pojo.ParamAction in project Taier by DTStack.

the class JobRestartDealer method restartJob.

private boolean restartJob(JobClient jobClient, BiConsumer<ScheduleJob, JobClient> saveRetryFunction) {
    ScheduleEngineJobCache jobCache = engineJobCacheService.getByJobId(jobClient.getJobId());
    if (jobCache == null) {
        LOGGER.info("jobId:{} restart but jobCache is null.", jobClient.getJobId());
        return false;
    }
    String jobInfo = jobCache.getJobInfo();
    try {
        ParamAction paramAction = PublicUtil.jsonStrToObject(jobInfo, ParamAction.class);
        jobClient.setSql(paramAction.getSqlText());
    } catch (IOException e) {
        LOGGER.error("jobId:{} restart but convert paramAction error: ", jobClient.getJobId(), e);
        return false;
    }
    // 添加到重试队列中
    boolean isAdd = jobDealer.addRestartJob(jobClient);
    if (isAdd) {
        String jobId = jobClient.getJobId();
        // 重试任务更改在zk的状态，统一做状态清理
        shardCache.updateLocalMemTaskStatus(jobId, TaskStatus.RESTARTING.getStatus());
        // 重试的任务不置为失败，waitengine
        ScheduleJob scheduleJob = scheduleJobService.getByJobId(jobClient.getJobId());
        if (saveRetryFunction != null) {
            saveRetryFunction.accept(scheduleJob, jobClient);
        } else {
            jobRetryRecord(scheduleJob, jobClient, null);
        }
        scheduleJobService.updateStatus(jobId, TaskStatus.RESTARTING.getStatus());
        LOGGER.info("jobId:{} update job status:{}.", jobId, TaskStatus.RESTARTING.getStatus());
        // update retryNum
        increaseJobRetryNum(jobClient.getJobId());
    }
    return isAdd;
}

Example 13 with ParamAction

use of com.dtstack.taier.pluginapi.pojo.ParamAction in project Taier by DTStack.

the class JobStatusDealer method dealJob.

private void dealJob(String jobId) throws Exception {
    ScheduleJob scheduleJob = scheduleJobService.getByJobId(jobId);
    ScheduleEngineJobCache engineJobCache = scheduleJobCacheService.getJobCacheByJobId(jobId);
    if (scheduleJob == null || engineJobCache == null || (StringUtils.isBlank(scheduleJob.getApplicationId()) && StringUtils.isBlank(scheduleJob.getEngineJobId()))) {
        shardCache.updateLocalMemTaskStatus(jobId, TaskStatus.CANCELED.getStatus());
        Integer status = TaskStatus.CANCELED.getStatus();
        String engineJobId = null;
        if (scheduleJob != null) {
            engineJobId = scheduleJob.getEngineJobId();
            if (TaskStatus.getStoppedStatus().contains(scheduleJob.getStatus())) {
                status = scheduleJob.getStatus();
            } else {
                scheduleJobService.updateJobStatusAndExecTime(jobId, status);
            }
        } else {
            scheduleJobService.updateJobStatusAndExecTime(jobId, status);
        }
        scheduleJobCacheService.deleteByJobId(jobId);
        LOGGER.info("jobId:{} set job finished, status:{}, scheduleJob is {} null, engineJobCache is {} null, engineJobId is {} blank.", jobId, status, scheduleJob == null ? "" : "not", engineJobCache == null ? "" : "not", engineJobId == null ? "" : "not");
    } else {
        String engineTaskId = scheduleJob.getEngineJobId();
        String appId = scheduleJob.getApplicationId();
        ParamAction paramAction = PublicUtil.jsonStrToObject(engineJobCache.getJobInfo(), ParamAction.class);
        Integer taskType = paramAction.getTaskType();
        Map<String, Object> pluginInfo = paramAction.getPluginInfo();
        JobIdentifier jobIdentifier = new JobIdentifier(engineTaskId, appId, jobId, scheduleJob.getTenantId(), taskType, TaskParamsUtils.parseDeployTypeByTaskParams(paramAction.getTaskParams(), scheduleJob.getComputeType()).getType(), null, MapUtils.isEmpty(pluginInfo) ? null : JSONObject.toJSONString(pluginInfo), paramAction.getComponentVersion());
        TaskStatus taskStatus = workerOperator.getJobStatus(jobIdentifier);
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("------ jobId:{} dealJob status:{}", jobId, taskStatus);
        }
        if (taskStatus != null) {
            taskStatus = checkNotFoundStatus(taskStatus, jobId);
            Integer status = taskStatus.getStatus();
            // 重试状态 先不更新状态
            boolean isRestart = jobRestartDealer.checkAndRestart(status, scheduleJob, engineJobCache, (job, client) -> ForkJoinPool.commonPool().execute(() -> {
                String engineLog = workerOperator.getEngineLog(jobIdentifier);
                jobRestartDealer.jobRetryRecord(job, client, engineLog);
            }));
            if (isRestart) {
                LOGGER.info("----- jobId:{} after dealJob status:{}", jobId, taskStatus);
                return;
            }
            shardCache.updateLocalMemTaskStatus(jobId, status);
            updateJobStatusWithPredicate(scheduleJob, jobId, status);
            // 数据的更新顺序，先更新job_cache，再更新engine_batch_job
            if (TaskStatus.getStoppedStatus().contains(status)) {
                jobLogDelayDealer(jobId, jobIdentifier, engineJobCache.getComputeType(), scheduleJob.getType());
                jobStatusFrequency.remove(jobId);
                scheduleJobCacheService.deleteByJobId(jobId);
                LOGGER.info("------ jobId:{} is stop status {} delete jobCache", jobId, status);
            }
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("------ jobId:{} after dealJob status:{}", jobId, taskStatus);
            }
        }
    }
}

Example 14 with ParamAction

use of com.dtstack.taier.pluginapi.pojo.ParamAction in project Taier by DTStack.

the class JobStopDealer method stopJob.

private StoppedStatus stopJob(JobElement jobElement) throws Exception {
    ScheduleEngineJobCache jobCache = engineJobCacheService.getByJobId(jobElement.jobId);
    ScheduleJob scheduleJob = scheduleJobService.lambdaQuery().eq(ScheduleJob::getJobId, jobElement.jobId).eq(ScheduleJob::getIsDeleted, Deleted.NORMAL.getStatus()).one();
    if (jobCache == null) {
        if (scheduleJob != null && TaskStatus.isStopped(scheduleJob.getStatus())) {
            LOGGER.info("jobId:{} stopped success, set job is STOPPED.", jobElement.jobId);
            return StoppedStatus.STOPPED;
        } else {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} jobCache is null, set job is MISSED.", jobElement.jobId);
            return StoppedStatus.MISSED;
        }
    } else if (null != scheduleJob && EJobCacheStage.unSubmitted().contains(jobCache.getStage())) {
        if (!TaskStatus.getWaitStatus().contains(scheduleJob.getStatus()) || EJobCacheStage.PRIORITY.getStage() != jobCache.getStage()) {
            this.removeMemStatusAndJobCache(jobCache.getJobId());
            LOGGER.info("jobId:{} is unsubmitted, set job is STOPPED.", jobElement.jobId);
            return StoppedStatus.STOPPED;
        } else {
            // 任务如果处于提交的状态过程中 但是stage由PRIORITY变更为SUBMITTED  直接删除会导致还是会提交到yarn上 占用资源
            LOGGER.info("jobId:{} is stopping.", jobCache.getJobId());
            return StoppedStatus.STOPPING;
        }
    } else {
        if (scheduleJob == null) {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} scheduleJob is null, set job is MISSED.", jobElement.jobId);
            return StoppedStatus.MISSED;
        } else if (TaskStatus.getStoppedAndNotFound().contains(scheduleJob.getStatus())) {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} and status:{} is StoppedAndNotFound, set job is STOPPED.", jobElement.jobId, scheduleJob.getStatus());
            return StoppedStatus.STOPPED;
        }
        ParamAction paramAction = PublicUtil.jsonStrToObject(jobCache.getJobInfo(), ParamAction.class);
        paramAction.setEngineTaskId(scheduleJob.getEngineJobId());
        paramAction.setApplicationId(scheduleJob.getApplicationId());
        JobClient jobClient = new JobClient(paramAction);
        jobClient.setForceCancel(jobElement.isForceCancel);
        if (StringUtils.isNotBlank(scheduleJob.getEngineJobId()) && !jobClient.getEngineTaskId().equals(scheduleJob.getEngineJobId())) {
            this.removeMemStatusAndJobCache(jobElement.jobId);
            LOGGER.info("jobId:{} stopped success, because of [difference engineJobId].", paramAction.getJobId());
            return StoppedStatus.STOPPED;
        }
        JobResult jobResult = workerOperator.stopJob(jobClient);
        if (jobResult.getCheckRetry()) {
            LOGGER.info("jobId:{} is retry.", paramAction.getJobId());
            return StoppedStatus.RETRY;
        } else {
            LOGGER.info("jobId:{} is stopping.", paramAction.getJobId());
            return StoppedStatus.STOPPING;
        }
    }
}

Example 15 with ParamAction

use of com.dtstack.taier.pluginapi.pojo.ParamAction in project Taier by DTStack.

the class JobClient method getParamAction.

public ParamAction getParamAction() {
    ParamAction action = new ParamAction();
    action.setSqlText(sql);
    action.setTaskParams(taskParams);
    action.setName(jobName);
    action.setJobId(jobId);
    action.setEngineTaskId(engineTaskId);
    action.setComputeType(computeType.getType());
    action.setExternalPath(externalPath);
    action.setExeArgs(classArgs);
    action.setGroupName(groupName);
    action.setGenerateTime(generateTime);
    action.setPriority(priority);
    action.setApplicationId(applicationId);
    action.setMaxRetryNum(maxRetryNum);
    action.setLackingCount(lackingCount);
    action.setTenantId(tenantId);
    action.setRetryIntervalTime(retryIntervalTime);
    action.setSubmitExpiredTime(submitExpiredTime);
    action.setComponentVersion(componentVersion);
    action.setTaskType(taskType);
    return action;
}