Examples with ScheduleTaskParamShade com.dtstack.taier.dao.dto.ScheduleTaskParamShade used on opensource projects

Example 1 with ScheduleTaskParamShade

use of com.dtstack.taier.dao.dto.ScheduleTaskParamShade in project Taier by DTStack.

the class ActionService method queryJobLog.

/**
 * 查看周期实例日志
 *
 * @param jobId    实例id
 * @param pageInfo 第几次重试日志
 * @return 日志信息
 */
public ReturnJobLogVO queryJobLog(String jobId, Integer pageInfo) {
    if (pageInfo == null) {
        pageInfo = 1;
    }
    // 查询周期实例
    ScheduleJob scheduleJob = jobService.lambdaQuery().eq(ScheduleJob::getJobId, jobId).eq(ScheduleJob::getIsDeleted, Deleted.NORMAL.getStatus()).one();
    if (scheduleJob == null) {
        throw new RdosDefineException("not find job,please contact the administrator");
    }
    // 取最新
    if (0 == pageInfo) {
        pageInfo = scheduleJob.getRetryNum();
    }
    ReturnJobLogVO jobLogVO = new ReturnJobLogVO();
    jobLogVO.setPageIndex(pageInfo);
    jobLogVO.setPageSize(scheduleJob.getRetryNum());
    // 如果RetryNum>1 说明实例已经进行了一次重试，所以取查询重试日志
    if (scheduleJob.getRetryNum() > 1) {
        // 查询重试日志
        ScheduleEngineJobRetry scheduleEngineJobRetry = jobRetryService.lambdaQuery().eq(ScheduleEngineJobRetry::getJobId, jobId).eq(ScheduleEngineJobRetry::getRetryNum, pageInfo).eq(ScheduleEngineJobRetry::getIsDeleted, Deleted.NORMAL.getStatus()).orderBy(true, false, ScheduleEngineJobRetry::getId).one();
        if (scheduleEngineJobRetry != null) {
            jobLogVO.setLogInfo(scheduleEngineJobRetry.getLogInfo());
            jobLogVO.setEngineLog(scheduleEngineJobRetry.getEngineLog());
        }
        jobLogVO.setPageIndex(pageInfo);
        jobLogVO.setPageSize(scheduleJob.getMaxRetryNum());
    } else {
        // 查询当前日志
        ScheduleJobExpand scheduleJobExpand = jobExpandService.lambdaQuery().eq(ScheduleJobExpand::getIsDeleted, Deleted.NORMAL.getStatus()).eq(ScheduleJobExpand::getJobId, jobId).one();
        if (scheduleJobExpand != null) {
            jobLogVO.setLogInfo(scheduleJobExpand.getLogInfo());
            jobLogVO.setEngineLog(scheduleJobExpand.getEngineLog());
        }
    }
    // 封装sql信息
    ScheduleTaskShade scheduleTaskShade = taskService.lambdaQuery().eq(ScheduleTaskShade::getTaskId, scheduleJob.getTaskId()).eq(ScheduleTaskShade::getIsDeleted, Deleted.NORMAL.getStatus()).one();
    if (null != scheduleTaskShade) {
        JSONObject shadeInfo = scheduleTaskShadeInfoService.getInfoJSON(scheduleTaskShade.getTaskId());
        String taskParams = shadeInfo.getString("taskParamsToReplace");
        List<ScheduleTaskParamShade> taskParamsToReplace = JSONObject.parseArray(taskParams, ScheduleTaskParamShade.class);
        String sqlText = scheduleTaskShade.getSqlText();
        if (EScheduleJobType.SYNC.getType().equals(scheduleTaskShade.getTaskType())) {
            sqlText = Base64Util.baseDecode(sqlText);
        }
        sqlText = JobParamReplace.paramReplace(sqlText, taskParamsToReplace, scheduleJob.getCycTime());
        jobLogVO.setSqlText(sqlText);
        Timestamp execStartTime = scheduleJob.getExecStartTime();
        Timestamp execEndTime = scheduleJob.getExecEndTime();
        if (EScheduleJobType.SYNC.getType().equals(scheduleTaskShade.getTaskType())) {
            String syncLog = null;
            try {
                syncLog = batchServerLogService.formatPerfLogInfo(scheduleJob.getEngineJobId(), scheduleJob.getJobId(), Optional.ofNullable(execStartTime).orElse(Timestamp.valueOf(LocalDateTime.now())).getTime(), Optional.ofNullable(execEndTime).orElse(Timestamp.valueOf(LocalDateTime.now())).getTime(), scheduleJob.getTenantId());
            } catch (Exception e) {
                LOGGER.error("queryJobLog {} sync log error", jobId, e);
            }
            jobLogVO.setSyncLog(syncLog);
        }
        if (EScheduleJobType.SPARK_SQL.getType().equals(scheduleTaskShade.getTaskType())) {
            jobLogVO.setDownLoadUrl(String.format(CommonConstant.DOWNLOAD_LOG, scheduleJob.getJobId(), scheduleJob.getTaskType(), scheduleJob.getTenantId()));
        }
    }
    return jobLogVO;
}

Example 2 with ScheduleTaskParamShade

use of com.dtstack.taier.dao.dto.ScheduleTaskParamShade in project Taier by DTStack.

the class ScheduleActionService method dealActionParam.

private void dealActionParam(Map<String, Object> actionParam, ScheduleTaskShade batchTask, ScheduleJob scheduleJob) throws Exception {
    IPipeline pipeline = null;
    String pipelineConfig = null;
    if (actionParam.containsKey(PipelineBuilder.pipelineKey)) {
        pipelineConfig = (String) actionParam.get(PipelineBuilder.pipelineKey);
        pipeline = PipelineBuilder.buildPipeline(pipelineConfig);
    } else if (EScheduleJobType.SPARK_SQL.getType().equals(batchTask.getTaskType())) {
        pipeline = PipelineBuilder.buildDefaultSqlPipeline();
    } else if (EScheduleJobType.SYNC.getType().equals(batchTask.getTaskType())) {
        pipeline = syncOperatorPipeline;
    }
    if (pipeline == null) {
        throw new RdosDefineException(ErrorCode.CONFIG_ERROR);
    }
    List<ScheduleTaskParamShade> taskParamsToReplace = JSONObject.parseArray((String) actionParam.get("taskParamsToReplace"), ScheduleTaskParamShade.class);
    Map<String, Object> pipelineInitMap = PipelineBuilder.getPipelineInitMap(pipelineConfig, scheduleJob, batchTask, taskParamsToReplace, (uploadPipelineMap) -> {
        // fill 文件上传的信息
        JSONObject pluginInfo = clusterService.pluginInfoJSON(batchTask.getTenantId(), batchTask.getTaskType(), null, null);
        String hdfsTypeName = componentService.buildHdfsTypeName(batchTask.getTenantId(), null);
        pluginInfo.put(ConfigConstant.TYPE_NAME_KEY, hdfsTypeName);
        uploadPipelineMap.put(UploadParamPipeline.pluginInfoKey, pluginInfo);
        uploadPipelineMap.put(UploadParamPipeline.fileUploadPathKey, environmentContext.getHdfsTaskPath());
    });
    pipeline.execute(actionParam, pipelineInitMap);
}

Example 3 with ScheduleTaskParamShade

use of com.dtstack.taier.dao.dto.ScheduleTaskParamShade in project Taier by DTStack.

the class UploadParamPipeline method pipeline.

@Override
public void pipeline(Map<String, Object> actionParam, Map<String, Object> pipelineParam) throws RdosDefineException {
    if (pipelineParam.containsKey(pipelineKey)) {
        return;
    }
    ScheduleTaskShade taskShade = (ScheduleTaskShade) pipelineParam.get(taskShadeKey);
    if (null == taskShade) {
        throw new RdosDefineException("upload param pipeline task shade can not be null");
    }
    ScheduleJob scheduleJob = (ScheduleJob) pipelineParam.get(scheduleJobKey);
    if (null == scheduleJob) {
        throw new RdosDefineException("upload param pipeline schedule job can not be null");
    }
    String fileUploadPath = (String) pipelineParam.get(fileUploadPathKey);
    if (StringUtils.isBlank(fileUploadPath)) {
        throw new RdosDefineException("upload param pipeline fileUploadPath can not be null");
    }
    WorkerOperator workerOperator = (WorkerOperator) pipelineParam.get(workOperatorKey);
    if (null == workerOperator) {
        throw new RdosDefineException("upload param pipeline workerOperator can not be null");
    }
    JSONObject pluginInfo = (JSONObject) pipelineParam.get(pluginInfoKey);
    if (null == pluginInfo) {
        throw new RdosDefineException("upload param pipeline pluginInfo can not be null");
    }
    @SuppressWarnings("unchecked") List<ScheduleTaskParamShade> taskParamShades = (List) pipelineParam.get(taskParamsToReplaceKey);
    String uploadPath = this.uploadSqlTextToHdfs((String) actionParam.get("sqlText"), taskShade.getTaskType(), taskShade.getName(), taskShade.getTenantId(), 0L, taskParamShades, scheduleJob.getCycTime(), fileUploadPath, pluginInfo, workerOperator, scheduleJob.getJobId());
    pipelineParam.put(pipelineKey, uploadPath);
}

Example 4 with ScheduleTaskParamShade

use of com.dtstack.taier.dao.dto.ScheduleTaskParamShade in project Taier by DTStack.

the class JobParamOperatorPipeline method pipeline.

@Override
public void pipeline(Map<String, Object> actionParam, Map<String, Object> pipelineParam) throws Exception {
    String urlKey = (String) super.getExecuteValue(actionParam, pipelineParam);
    if (StringUtils.isNotBlank(urlKey)) {
        @SuppressWarnings("unchecked") List<ScheduleTaskParamShade> taskParamShades = (List) pipelineParam.get(taskParamsToReplaceKey);
        ScheduleJob scheduleJob = (ScheduleJob) pipelineParam.get(scheduleJobKey);
        if (null == scheduleJob) {
            throw new RdosDefineException("upload param pipeline schedule job can not be null");
        }
        pipelineParam.put(pipelineKey, JobParamReplace.paramReplace(urlKey, taskParamShades, scheduleJob.getCycTime()));
    }
}

Example 5 with ScheduleTaskParamShade

use of com.dtstack.taier.dao.dto.ScheduleTaskParamShade in project Taier by DTStack.

the class SyncOperatorPipeline method pipeline.

@Override
public void pipeline(Map<String, Object> actionParam, Map<String, Object> pipelineParam) throws Exception {
    ScheduleTaskShade taskShade = (ScheduleTaskShade) pipelineParam.get(taskShadeKey);
    ScheduleJob scheduleJob = (ScheduleJob) pipelineParam.get(scheduleJobKey);
    List<ScheduleTaskParamShade> taskParamShades = (List) pipelineParam.get(taskParamsToReplaceKey);
    String taskParams = (String) actionParam.get("taskParams");
    String job = (String) actionParam.get("job");
    EDeployMode deployMode = TaskParamsUtils.parseDeployTypeByTaskParams(taskParams, taskShade.getComputeType());
    job = this.replaceSyncJobString(actionParam, taskShade, scheduleJob, taskParamShades, job, deployMode);
    // 构造savepoint参数
    String savepointArgs = null;
    String taskExeArgs = null;
    if (isRestore(job)) {
        String savepointPath = this.getSavepointPath(taskShade.getTenantId(), deployMode, taskShade.getComponentVersion());
        savepointArgs = buildSyncTaskExecArgs(savepointPath, taskParams);
        taskParams += String.format(" \n %s=%s", KEY_OPEN_CHECKPOINT, Boolean.TRUE);
    }
    job = URLEncoder.encode(job.replace(CommonConstant.JOB_ID, scheduleJob.getJobId()), Charsets.UTF_8.name());
    taskExeArgs = String.format(JOB_ARGS_TEMPLATE, scheduleJob.getJobName(), job);
    if (savepointArgs != null) {
        taskExeArgs += " " + savepointArgs;
    }
    actionParam.put("exeArgs", taskExeArgs);
    actionParam.put("taskParams", taskParams);
}