Examples with Communication com.alibaba.datax.core.statistics.communication.Communication used on opensource projects

Example 11 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class ProcessInnerReporterTest method testReportJobCommunication.

@Test
public void testReportJobCommunication() {
    Long jobId = 0L;
    Communication communication = new Communication();
    ProcessInnerReporter processInnerReporter = new ProcessInnerReporter();
    processInnerReporter.reportJobCommunication(jobId, communication);
    System.out.println("this function do noting");
}

Example 12 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class ProcessInnerReporterTest method testReportTGCommunication.

@Test
public void testReportTGCommunication() throws NoSuchFieldException, IllegalAccessException {
    Integer taskGroupId = 1;
    Communication communication = new Communication();
    communication.setState(State.SUBMITTING);
    ConcurrentHashMap<Integer, Communication> map = new ConcurrentHashMap<Integer, Communication>();
    map.put(taskGroupId, communication);
    ReflectUtil.setField(new LocalTGCommunicationManager(), "taskGroupCommunicationMap", map);
    ProcessInnerReporter processInnerReporter = new ProcessInnerReporter();
    Communication updateCommunication = new Communication();
    updateCommunication.setState(State.WAITING);
    processInnerReporter.reportTGCommunication(taskGroupId, updateCommunication);
    Assert.assertEquals(map.get(taskGroupId).getState(), State.WAITING);
}

Example 13 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger.

@Test
public void test_BufferExchanger() {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    Channel channel = new MemoryChannel(configuration);
    channel.setCommunication(new Communication());
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    BufferedRecordExchanger recordExchanger = new BufferedRecordExchanger(channel, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger.sendToWriter(record);
    }
    recordExchanger.flush();
    channel.close();
    int counter = 0;
    while ((record = recordExchanger.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(capacity == counter);
}

Example 14 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class RecordExchangerTest method test_Exchanger.

@Test
public void test_Exchanger() {
    Channel channel = new MemoryChannel(configuration);
    channel.setCommunication(new Communication());
    int capacity = 10;
    Record record = null;
    RecordExchanger recordExchanger = new RecordExchanger(1, 0, channel, new Communication(), null, null);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger.sendToWriter(record);
    }
    System.out.println("byteSize=" + record.getByteSize());
    System.out.println("meorySize=" + record.getMemorySize());
    channel.close();
    int counter = 0;
    while ((record = recordExchanger.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    Assert.assertTrue(capacity == counter);
}

Example 15 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class JobContainerTest method testErrorLimitCountCheck.

@Test(expected = Exception.class)
public void testErrorLimitCountCheck() throws Exception {
    this.configuration.remove(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT_PERCENT);
    this.configuration.set(CoreConstant.DATAX_JOB_SETTING_ERRORLIMIT_RECORD, 1);
    JobContainer jobContainer = new JobContainer(this.configuration);
    Communication communication = new Communication();
    communication.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
    communication.setLongCounter(CommunicationTool.WRITE_RECEIVED_RECORDS, 98);
    communication.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 2);
    //        LocalTaskGroupCommunicationManager.updateTaskGroupCommunication(0, communication);
    Method initMethod = jobContainer.getClass().getDeclaredMethod("checkLimit");
    initMethod.setAccessible(true);
    initMethod.invoke(jobContainer);
    initMethod.setAccessible(false);
}