Examples with Communication com.alibaba.datax.core.statistics.communication.Communication used on opensource projects

Example 6 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class TaskGroupContainer method markCommunicationFailed.

private void markCommunicationFailed(Integer taskId) {
    Communication communication = containerCommunicator.getCommunication(taskId);
    communication.setState(State.FAILED);
}

Example 7 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class TaskGroupContainerTest method initConfiguration.

private void initConfiguration(Configuration configuration) {
    int channelNumber = 5;
    taskNumber = channelNumber + 3;
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, 0);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_SLEEPINTERVAL, 200);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_REPORTINTERVAL, 1000);
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, channelNumber);
    Configuration jobContent = configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    for (int i = 0; i < this.taskNumber; i++) {
        Configuration newJobContent = jobContent.clone();
        newJobContent.set(CoreConstant.TASK_ID, i);
        jobContents.add(newJobContent);
    }
    configuration.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    LocalTGCommunicationManager.clear();
    LocalTGCommunicationManager.registerTaskGroupCommunication(1, new Communication());
}

Example 8 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class ErrorRecordLimitTest method testCheckRecordLimit3.

@Test
public void testCheckRecordLimit3() throws Exception {
    // 百分数无效
    ErrorRecordChecker errLimit = new ErrorRecordChecker(1L, 0.05);
    errLimit.checkPercentageLimit(new Communication() {

        {
            this.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
            this.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 50);
        }
    });
}

Example 9 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class StandAloneSchedulerTest method testSchedule.

@Test
public void testSchedule() throws NoSuchFieldException, IllegalAccessException {
    int taskNumber = 10;
    List<Configuration> jobList = new ArrayList<Configuration>();
    List<Configuration> internal = new ArrayList<Configuration>();
    int randomSize = 20;
    int length = RandomUtils.nextInt(0, randomSize) + 1;
    for (int i = 0; i < length; i++) {
        internal.add(Configuration.newDefault());
    }
    LocalTGCommunicationManager.clear();
    for (int i = 0; i < taskNumber; i++) {
        Configuration configuration = Configuration.newDefault();
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_REPORTINTERVAL, 11);
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_ID, 0);
        configuration.set(CoreConstant.DATAX_JOB_CONTENT, internal);
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_JOB_MODE, ExecuteMode.STANDALONE.getValue());
        configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, i);
        jobList.add(configuration);
        LocalTGCommunicationManager.registerTaskGroupCommunication(i, new Communication());
    }
    StandAloneJobContainerCommunicator standAloneJobContainerCommunicator = PowerMockito.mock(StandAloneJobContainerCommunicator.class);
    ProcessInnerScheduler scheduler = PowerMockito.spy(new StandAloneScheduler(standAloneJobContainerCommunicator));
    PowerMockito.doNothing().when(scheduler).startAllTaskGroup(anyListOf(Configuration.class));
    Communication communication = new Communication();
    communication.setState(State.SUCCEEDED);
    PowerMockito.when(standAloneJobContainerCommunicator.collect()).thenReturn(communication);
    PowerMockito.doNothing().when(standAloneJobContainerCommunicator).report(communication);
    scheduler.schedule(jobList);
}

Example 10 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class ProcessInnerCollectorTest method testCollectFromTaskGroup.

@Test
public void testCollectFromTaskGroup() throws NoSuchFieldException, IllegalAccessException {
    Integer taskGroupId_1 = 1;
    Integer taskGroupId_2 = 2;
    Communication communication_1 = new Communication();
    communication_1.setLongCounter("totalBytes", 888);
    Communication communication_2 = new Communication();
    communication_2.setLongCounter("totalBytes", 112);
    ConcurrentHashMap<Integer, Communication> taskGroupCommunicationMap = new ConcurrentHashMap<Integer, Communication>();
    taskGroupCommunicationMap.put(taskGroupId_1, communication_1);
    taskGroupCommunicationMap.put(taskGroupId_2, communication_2);
    ReflectUtil.setField(new LocalTGCommunicationManager(), "taskGroupCommunicationMap", taskGroupCommunicationMap);
    ProcessInnerCollector processInnerCollector = new ProcessInnerCollector(0L);
    Communication comm = processInnerCollector.collectFromTaskGroup();
    Assert.assertTrue(comm.getLongCounter("totalBytes") == 1000);
    System.out.println(comm);
}