Examples with RdosDefineException com.dtstack.taier.common.exception.RdosDefineException used on opensource projects

Example 11 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class ConsoleComponentService method getComponentStore.

public List<Component> getComponentStore(String clusterName, Integer componentType) {
    Cluster cluster = clusterMapper.getByClusterName(clusterName);
    if (null == cluster) {
        throw new RdosDefineException("Cluster does not exist");
    }
    List<Component> components = new ArrayList<>();
    Component hdfs = componentMapper.getByClusterIdAndComponentType(cluster.getId(), EComponentType.HDFS.getTypeCode(), null, null);
    if (null != hdfs) {
        components.add(hdfs);
    }
    return components;
}

Example 12 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class TenantService method checkClusterCanUse.

public void checkClusterCanUse(String clusterName) throws Exception {
    List<ComponentMultiTestResult> testConnectionVO = consoleComponentService.testConnects(clusterName);
    boolean canUse = true;
    StringBuilder msg = new StringBuilder();
    msg.append("此集群不可用,测试连通性为通过：\n");
    for (ComponentMultiTestResult testResult : testConnectionVO) {
        EComponentType componentType = EComponentType.getByCode(testResult.getComponentTypeCode());
        if (!EComponentType.notCheckComponent.contains(componentType) && !testResult.getResult()) {
            canUse = false;
            msg.append("组件:").append(componentType.getName()).append(" ").append(JSON.toJSONString(testResult.getErrorMsg())).append("\n");
        }
    }
    if (!canUse) {
        throw new RdosDefineException(msg.toString());
    }
}

Example 13 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class UpstreamDependencyHandler method getJobKey.

/**
 * 获得jobKey
 *
 * @param scheduleTaskShade 任务
 * @param currentDate 当前时间
 * @return jobKey
 */
public String getJobKey(ScheduleTaskShade scheduleTaskShade, Date currentDate) throws Exception {
    ScheduleCorn corn = ScheduleConfManager.parseFromJson(scheduleTaskShade.getScheduleConf());
    // 上一个周期
    Date lastDate = corn.isMatch(currentDate) ? currentDate : corn.last(currentDate);
    String lastDateStr = DateUtil.getDate(corn.isMatch(currentDate) ? currentDate : corn.last(currentDate), DateUtil.STANDARD_DATETIME_FORMAT);
    if (StringUtils.isBlank(lastDateStr)) {
        throw new RdosDefineException("no find upstream task of last cycle");
    }
    String jobKey = JobKeyUtils.generateJobKey(keyPreStr, scheduleTaskShade.getTaskId(), lastDateStr);
    return needCreateKey(lastDate, currentDate, jobKey);
}

Example 14 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class CycleJobBuilder method buildTaskJobGraph.

public void buildTaskJobGraph(String triggerDay) {
    if (environmentContext.getJobGraphBuilderSwitch()) {
        return;
    }
    lock.lock();
    try {
        String triggerTimeStr = triggerDay + " 00:00:00";
        Timestamp triggerTime = Timestamp.valueOf(triggerTimeStr);
        boolean hasBuild = jobGraphTriggerService.checkHasBuildJobGraph(triggerTime);
        if (hasBuild) {
            LOGGER.info("trigger Day {} has build so break", triggerDay);
            return;
        }
        // 1. 获得今天预计要生成的所有周期实例
        Integer totalTask = getTotalTask();
        LOGGER.info("{} need build job : {}", triggerTimeStr, totalTask);
        if (totalTask <= 0) {
            saveJobGraph(triggerDay);
            return;
        }
        clearInterruptJob(triggerTime);
        // 2. 切割总数 限制 thread 并发
        int totalBatch = totalTask / environmentContext.getJobLimitSize();
        if (totalTask % environmentContext.getJobLimitSize() != 0) {
            totalBatch++;
        }
        Semaphore sph = new Semaphore(environmentContext.getMaxTaskBuildThread());
        CountDownLatch ctl = new CountDownLatch(totalBatch);
        AtomicJobSortWorker sortWorker = new AtomicJobSortWorker();
        // 3. 查询db多线程生成周期实例
        Long startId = 0L;
        for (int i = 0; i < totalBatch; i++) {
            // 取50个任务
            final List<ScheduleTaskShade> batchTaskShades = scheduleTaskService.listRunnableTask(startId, Lists.newArrayList(EScheduleStatus.NORMAL.getVal(), EScheduleStatus.FREEZE.getVal()), environmentContext.getJobLimitSize());
            // 如果取出来的任务集合是空的
            if (CollectionUtils.isEmpty(batchTaskShades)) {
                continue;
            }
            startId = batchTaskShades.get(batchTaskShades.size() - 1).getId();
            LOGGER.info("job-number:{} startId:{}", i, startId);
            try {
                sph.acquire();
                jobGraphBuildPool.submit(() -> {
                    try {
                        for (ScheduleTaskShade batchTaskShade : batchTaskShades) {
                            try {
                                List<ScheduleJobDetails> scheduleJobDetails = RetryUtil.executeWithRetry(() -> buildJob(batchTaskShade, triggerDay, sortWorker), environmentContext.getBuildJobErrorRetry(), 200, false);
                                // 插入周期实例
                                savaJobList(scheduleJobDetails);
                            } catch (Throwable e) {
                                LOGGER.error("build task failure taskId:{} apptype:{}", batchTaskShade.getTaskId(), null, e);
                            }
                        }
                    } catch (Throwable e) {
                        LOGGER.error("!!! buildTaskJobGraph  build job error !!!", e);
                    } finally {
                        sph.release();
                        ctl.countDown();
                    }
                });
            } catch (Throwable e) {
                LOGGER.error("[acquire pool error]:", e);
                throw new RdosDefineException(e);
            }
        }
        ctl.await();
        // 循环已经结束，说明周期实例已经全部生成了
        saveJobGraph(triggerDay);
    } catch (Exception e) {
        LOGGER.error("buildTaskJobGraph ！！！", e);
    } finally {
        LOGGER.info("buildTaskJobGraph exit & unlock ...");
        lock.unlock();
    }
}

Example 15 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class ScheduleConfManager method parseFromJson.

public static ScheduleCorn parseFromJson(String scheduleConf) throws IOException, ParseException {
    ScheduleConf scheduleConfBean = JSON.parseObject(scheduleConf, ScheduleConf.class);
    // 校验必要参数
    checkConf(scheduleConf, scheduleConfBean);
    // 获得对应的corn表达式
    int periodType = scheduleConfBean.getPeriodType();
    IScheduleConfParser scheduleCron;
    if (periodType == ESchedulePeriodType.MONTH.getVal()) {
        scheduleCron = new ScheduleCronMonthParser();
    } else if (periodType == ESchedulePeriodType.WEEK.getVal()) {
        scheduleCron = new ScheduleCronWeekParser();
    } else if (periodType == ESchedulePeriodType.DAY.getVal()) {
        scheduleCron = new ScheduleCronDayParser();
    } else if (periodType == ESchedulePeriodType.HOUR.getVal()) {
        scheduleCron = new ScheduleCronHourParser();
    } else if (periodType == ESchedulePeriodType.MIN.getVal()) {
        scheduleCron = new ScheduleCronMinParser();
    } else if (periodType == ESchedulePeriodType.CUSTOM.getVal()) {
        scheduleCron = new ScheduleCronCustomParser();
    } else {
        throw new RdosDefineException("not support period type!");
    }
    String cron = scheduleCron.parse(scheduleConfBean);
    // 返回corn对象
    ScheduleCorn corn = new ScheduleCorn();
    corn.setScheduleConf(scheduleConfBean);
    corn.setCron(cron);
    return corn;
}