Examples with TaskPluginCollector com.alibaba.datax.common.plugin.TaskPluginCollector used on opensource projects

Example 1 with TaskPluginCollector

use of com.alibaba.datax.common.plugin.TaskPluginCollector in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger.

@Test
public void test_BufferExchanger() {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    Channel channel = new MemoryChannel(configuration);
    channel.setCommunication(new Communication());
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    BufferedRecordExchanger recordExchanger = new BufferedRecordExchanger(channel, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger.sendToWriter(record);
    }
    recordExchanger.flush();
    channel.close();
    int counter = 0;
    while ((record = recordExchanger.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(capacity == counter);
}

Example 2 with TaskPluginCollector

use of com.alibaba.datax.common.plugin.TaskPluginCollector in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_单条超过buffer的脏数据.

@Test
public void test_BufferExchanger_单条超过buffer的脏数据() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    //测试单挑记录超过buffer大小
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 3);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    Channel channel2 = new MemoryChannel(configuration);
    channel2.setCommunication(new Communication());
    BufferedRecordExchanger recordExchanger2 = new BufferedRecordExchanger(channel2, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger2.sendToWriter(record);
    }
    ArgumentCaptor<Record> rgArg = ArgumentCaptor.forClass(Record.class);
    ArgumentCaptor<Exception> eArg = ArgumentCaptor.forClass(Exception.class);
    verify(pluginCollector, times(10)).collectDirtyRecord(rgArg.capture(), eArg.capture());
    recordExchanger2.flush();
    channel2.close();
    int counter = 0;
    while ((record = recordExchanger2.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(counter == 0);
}

Example 3 with TaskPluginCollector

use of com.alibaba.datax.common.plugin.TaskPluginCollector in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_每条大小刚好是buffersize.

@Test
public void test_BufferExchanger_每条大小刚好是buffersize() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 229);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    final int capacity = 10;
    Record record = null;
    //测试单挑记录超过buffer大小
    Channel channel3 = new MemoryChannel(configuration);
    channel3.setCommunication(new Communication());
    final BufferedRecordExchanger recordExchangerWriter = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger recordExchangerReader = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger spy1 = spy(recordExchangerWriter);
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int counter = 0;
            Record record;
            while ((record = recordExchangerReader.getFromReader()) != null) {
                System.out.println(record.getColumn(0).toString());
                Assert.assertTrue(record.getColumn(0).asLong() == counter);
                counter++;
            }
            System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
            Assert.assertTrue(capacity == counter);
        }
    });
    t.start();
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        spy1.sendToWriter(record);
    }
    spy1.flush();
    channel3.close();
    t.join();
    verify(spy1, times(10)).flush();
}

Example 4 with TaskPluginCollector

use of com.alibaba.datax.common.plugin.TaskPluginCollector in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_不满32条到达buffer大小.

@Test
public void test_BufferExchanger_不满32条到达buffer大小() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 500);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    final int capacity = 10;
    Record record = null;
    //测试单挑记录超过buffer大小
    Channel channel3 = new MemoryChannel(configuration);
    channel3.setCommunication(new Communication());
    final BufferedRecordExchanger recordExchangerWriter = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger recordExchangerReader = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger spy1 = spy(recordExchangerWriter);
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int counter = 0;
            Record record;
            while ((record = recordExchangerReader.getFromReader()) != null) {
                System.out.println(record.getColumn(0).toString());
                Assert.assertTrue(record.getColumn(0).asLong() == counter);
                counter++;
            }
            System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
            Assert.assertTrue(capacity == counter);
        }
    });
    t.start();
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        spy1.sendToWriter(record);
    }
    spy1.flush();
    channel3.close();
    t.join();
    verify(spy1, times(5)).flush();
}