Examples with Communication com.alibaba.datax.core.statistics.communication.Communication used on opensource projects

Example 31 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class TaskMonitorTest method testNormal.

@Test
public void testNormal() throws Exception {
    //register task
    long ttl = System.currentTimeMillis();
    Communication communication1 = new Communication();
    taskMonitor.registerTask(1, communication1);
    TaskMonitor.TaskCommunication taskCommunication1 = taskMonitor.getTaskCommunication(1);
    Assert.assertEquals(taskCommunication1.getLastAllReadRecords(), 0L);
    Assert.assertEquals(this.tasks.size(), 1);
    Assert.assertTrue(taskCommunication1.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication1.getTtl() >= ttl);
    // report 没有任何变化的communication
    long oldTS = taskCommunication1.getLastUpdateComunicationTS();
    long oldTTL = taskCommunication1.getTtl();
    Thread.sleep(1000);
    taskMonitor.report(1, communication1);
    TaskMonitor.TaskCommunication taskCommunication1_1 = taskMonitor.getTaskCommunication(1);
    Assert.assertEquals(taskCommunication1_1.getLastAllReadRecords(), 0L);
    Assert.assertEquals(taskCommunication1_1.getLastUpdateComunicationTS(), oldTS);
    Assert.assertTrue(taskCommunication1_1.getTtl() > oldTTL);
    // report 已经finish的communication
    Communication communication2 = new Communication();
    communication2.setState(State.KILLED);
    taskMonitor.registerTask(2, communication2);
    Assert.assertEquals(this.tasks.size(), 1);
    // report 另一个communication
    Communication communication3 = new Communication();
    taskMonitor.registerTask(3, communication3);
    Assert.assertEquals(this.tasks.size(), 2);
    System.out.println(this.tasks);
    //report communication
    ttl = System.currentTimeMillis();
    communication1.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 100);
    communication3.setLongCounter(CommunicationTool.READ_FAILED_RECORDS, 10);
    taskMonitor.report(1, communication1);
    taskMonitor.report(3, communication3);
    taskCommunication1 = taskMonitor.getTaskCommunication(1);
    Assert.assertEquals(taskCommunication1.getLastAllReadRecords(), 100L);
    Assert.assertEquals(this.tasks.size(), 2);
    Assert.assertTrue(taskCommunication1.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication1.getTtl() >= ttl);
    TaskMonitor.TaskCommunication taskCommunication3 = taskMonitor.getTaskCommunication(3);
    Assert.assertEquals(taskCommunication3.getLastAllReadRecords(), 10L);
    Assert.assertEquals(this.tasks.size(), 2);
    Assert.assertTrue(taskCommunication3.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication3.getTtl() >= ttl);
    //继续report
    ttl = System.currentTimeMillis();
    communication1.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 1001);
    communication3.setLongCounter(CommunicationTool.READ_FAILED_RECORDS, 101);
    taskMonitor.report(1, communication1);
    taskMonitor.report(3, communication3);
    taskCommunication1 = taskMonitor.getTaskCommunication(1);
    Assert.assertEquals(taskCommunication1.getLastAllReadRecords(), 1001L);
    Assert.assertEquals(this.tasks.size(), 2);
    Assert.assertTrue(taskCommunication1.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication1.getTtl() >= ttl);
    taskCommunication3 = taskMonitor.getTaskCommunication(3);
    Assert.assertEquals(taskCommunication3.getLastAllReadRecords(), 101L);
    Assert.assertEquals(this.tasks.size(), 2);
    Assert.assertTrue(taskCommunication3.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication3.getTtl() >= ttl);
    // 设置EXPIRED_TIME
    Field EXPIRED_TIME = taskMonitor.getClass().getDeclaredField("EXPIRED_TIME");
    EXPIRED_TIME.setAccessible(true);
    EXPIRED_TIME.set(null, 1000);
    Thread.sleep(2000);
    //超时没有变更
    taskMonitor.report(1, communication1);
    System.out.println(communication1.getCounter());
    System.out.println(communication1.getThrowable());
    System.out.println(communication1.getThrowableMessage());
    System.out.println(communication1.getState());
    Assert.assertTrue(communication1.getThrowableMessage().contains("任务hung住，Expired"));
    Assert.assertEquals(communication1.getState(), State.FAILED);
    // communicatio1 已经fail， communication3 在超时后进行变更，update正常
    ttl = System.currentTimeMillis();
    communication1.setLongCounter(CommunicationTool.READ_SUCCEED_RECORDS, 2001);
    communication3.setLongCounter(CommunicationTool.READ_FAILED_RECORDS, 201);
    taskMonitor.report(1, communication1);
    taskMonitor.report(3, communication3);
    taskCommunication1 = taskMonitor.getTaskCommunication(1);
    Assert.assertEquals(taskCommunication1.getLastAllReadRecords(), 1001L);
    Assert.assertEquals(this.tasks.size(), 2);
    Assert.assertTrue(communication1.getThrowableMessage().contains("任务hung住，Expired"));
    Assert.assertEquals(communication1.getState(), State.FAILED);
    taskCommunication3 = taskMonitor.getTaskCommunication(3);
    Assert.assertEquals(taskCommunication3.getLastAllReadRecords(), 201L);
    Assert.assertEquals(this.tasks.size(), 2);
    Assert.assertTrue(taskCommunication3.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication3.getTtl() >= ttl);
    //remove 1
    taskMonitor.removeTask(1);
    Assert.assertEquals(this.tasks.size(), 1);
    //remove 3
    taskMonitor.removeTask(3);
    Assert.assertEquals(this.tasks.size(), 0);
    // 没有register communication3 直接report
    ttl = System.currentTimeMillis();
    communication3.setLongCounter(CommunicationTool.READ_FAILED_RECORDS, 301);
    taskMonitor.report(3, communication3);
    taskCommunication3 = taskMonitor.getTaskCommunication(3);
    Assert.assertEquals(taskCommunication3.getLastAllReadRecords(), 301L);
    Assert.assertEquals(this.tasks.size(), 1);
    Assert.assertTrue(taskCommunication3.getLastUpdateComunicationTS() >= ttl);
    Assert.assertTrue(taskCommunication3.getTtl() >= ttl);
}

Example 32 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class Consumer method before.

@Before
public void before() {
    System.out.println(ConfigurationProducer.produce().toJSON());
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 0);
    this.channel = new MemoryChannel(configuration);
    this.channel.setCommunication(new Communication());
}

Example 33 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_单条超过buffer的脏数据.

@Test
public void test_BufferExchanger_单条超过buffer的脏数据() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    //测试单挑记录超过buffer大小
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 3);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    Channel channel2 = new MemoryChannel(configuration);
    channel2.setCommunication(new Communication());
    BufferedRecordExchanger recordExchanger2 = new BufferedRecordExchanger(channel2, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger2.sendToWriter(record);
    }
    ArgumentCaptor<Record> rgArg = ArgumentCaptor.forClass(Record.class);
    ArgumentCaptor<Exception> eArg = ArgumentCaptor.forClass(Exception.class);
    verify(pluginCollector, times(10)).collectDirtyRecord(rgArg.capture(), eArg.capture());
    recordExchanger2.flush();
    channel2.close();
    int counter = 0;
    while ((record = recordExchanger2.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(counter == 0);
}

Example 34 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_每条大小刚好是buffersize.

@Test
public void test_BufferExchanger_每条大小刚好是buffersize() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 229);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    final int capacity = 10;
    Record record = null;
    //测试单挑记录超过buffer大小
    Channel channel3 = new MemoryChannel(configuration);
    channel3.setCommunication(new Communication());
    final BufferedRecordExchanger recordExchangerWriter = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger recordExchangerReader = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger spy1 = spy(recordExchangerWriter);
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int counter = 0;
            Record record;
            while ((record = recordExchangerReader.getFromReader()) != null) {
                System.out.println(record.getColumn(0).toString());
                Assert.assertTrue(record.getColumn(0).asLong() == counter);
                counter++;
            }
            System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
            Assert.assertTrue(capacity == counter);
        }
    });
    t.start();
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        spy1.sendToWriter(record);
    }
    spy1.flush();
    channel3.close();
    t.join();
    verify(spy1, times(10)).flush();
}

Example 35 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_不满32条到达buffer大小.

@Test
public void test_BufferExchanger_不满32条到达buffer大小() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 500);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    final int capacity = 10;
    Record record = null;
    //测试单挑记录超过buffer大小
    Channel channel3 = new MemoryChannel(configuration);
    channel3.setCommunication(new Communication());
    final BufferedRecordExchanger recordExchangerWriter = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger recordExchangerReader = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger spy1 = spy(recordExchangerWriter);
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int counter = 0;
            Record record;
            while ((record = recordExchangerReader.getFromReader()) != null) {
                System.out.println(record.getColumn(0).toString());
                Assert.assertTrue(record.getColumn(0).asLong() == counter);
                counter++;
            }
            System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
            Assert.assertTrue(capacity == counter);
        }
    });
    t.start();
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        spy1.sendToWriter(record);
    }
    spy1.flush();
    channel3.close();
    t.join();
    verify(spy1, times(5)).flush();
}