Examples with ErrorRecordChecker com.alibaba.datax.core.util.ErrorRecordChecker used on opensource projects

Example 1 with ErrorRecordChecker

use of com.alibaba.datax.core.util.ErrorRecordChecker in project DataX by alibaba.

the class AbstractScheduler method schedule.

public void schedule(List<Configuration> configurations) {
    Validate.notNull(configurations, "scheduler配置不能为空");
    int jobReportIntervalInMillSec = configurations.get(0).getInt(CoreConstant.DATAX_CORE_CONTAINER_JOB_REPORTINTERVAL, 30000);
    int jobSleepIntervalInMillSec = configurations.get(0).getInt(CoreConstant.DATAX_CORE_CONTAINER_JOB_SLEEPINTERVAL, 10000);
    this.jobId = configurations.get(0).getLong(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID);
    errorLimit = new ErrorRecordChecker(configurations.get(0));
    /**
         * 给 taskGroupContainer 的 Communication 注册
         */
    this.containerCommunicator.registerCommunication(configurations);
    int totalTasks = calculateTaskCount(configurations);
    startAllTaskGroup(configurations);
    Communication lastJobContainerCommunication = new Communication();
    long lastReportTimeStamp = System.currentTimeMillis();
    try {
        while (true) {
            /**
                 * step 1: collect job stat
                 * step 2: getReport info, then report it
                 * step 3: errorLimit do check
                 * step 4: dealSucceedStat();
                 * step 5: dealKillingStat();
                 * step 6: dealFailedStat();
                 * step 7: refresh last job stat, and then sleep for next while
                 *
                 * above steps, some ones should report info to DS
                 *
                 */
            Communication nowJobContainerCommunication = this.containerCommunicator.collect();
            nowJobContainerCommunication.setTimestamp(System.currentTimeMillis());
            LOG.debug(nowJobContainerCommunication.toString());
            //汇报周期
            long now = System.currentTimeMillis();
            if (now - lastReportTimeStamp > jobReportIntervalInMillSec) {
                Communication reportCommunication = CommunicationTool.getReportCommunication(nowJobContainerCommunication, lastJobContainerCommunication, totalTasks);
                this.containerCommunicator.report(reportCommunication);
                lastReportTimeStamp = now;
                lastJobContainerCommunication = nowJobContainerCommunication;
            }
            errorLimit.checkRecordLimit(nowJobContainerCommunication);
            if (nowJobContainerCommunication.getState() == State.SUCCEEDED) {
                LOG.info("Scheduler accomplished all tasks.");
                break;
            }
            if (isJobKilling(this.getJobId())) {
                dealKillingStat(this.containerCommunicator, totalTasks);
            } else if (nowJobContainerCommunication.getState() == State.FAILED) {
                dealFailedStat(this.containerCommunicator, nowJobContainerCommunication.getThrowable());
            }
            Thread.sleep(jobSleepIntervalInMillSec);
        }
    } catch (InterruptedException e) {
        // 以 failed 状态退出
        LOG.error("捕获到InterruptedException异常!", e);
        throw DataXException.asDataXException(FrameworkErrorCode.RUNTIME_ERROR, e);
    }
}

Example 2 with ErrorRecordChecker

use of com.alibaba.datax.core.util.ErrorRecordChecker in project DataX by alibaba.

the class ErrorRecordLimitTest method testCheckRecordLimit3.

@Test
public void testCheckRecordLimit3() throws Exception {
    // 百分数无效
    ErrorRecordChecker errLimit = new ErrorRecordChecker(1L, 0.05);
    errLimit.checkPercentageLimit(new Communication() {

        {
            this.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
            this.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 50);
        }
    });
}

Example 3 with ErrorRecordChecker

use of com.alibaba.datax.core.util.ErrorRecordChecker in project DataX by alibaba.

the class ErrorRecordLimitTest method testCheckRecordLimit2.

@Test
public void testCheckRecordLimit2() throws Exception {
    ErrorRecordChecker errLimit = new ErrorRecordChecker(1L, 0.5);
    errLimit.checkRecordLimit(new Communication() {

        {
            this.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 1);
        }
    });
}

Example 4 with ErrorRecordChecker

use of com.alibaba.datax.core.util.ErrorRecordChecker in project DataX by alibaba.

the class ErrorRecordLimitTest method testCheckRecordLimit.

@Test(expected = DataXException.class)
public void testCheckRecordLimit() throws Exception {
    ErrorRecordChecker errLimit = new ErrorRecordChecker(0L, 0.5);
    errLimit.checkRecordLimit(new Communication() {

        {
            this.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 1);
        }
    });
}