Examples with LongColumn com.alibaba.datax.common.element.LongColumn used on opensource projects

Example 6 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class RecordExchangerTest method test_Exchanger.

@Test
public void test_Exchanger() {
    Channel channel = new MemoryChannel(configuration);
    channel.setCommunication(new Communication());
    int capacity = 10;
    Record record = null;
    RecordExchanger recordExchanger = new RecordExchanger(1, 0, channel, new Communication(), null, null);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger.sendToWriter(record);
    }
    System.out.println("byteSize=" + record.getByteSize());
    System.out.println("meorySize=" + record.getMemorySize());
    channel.close();
    int counter = 0;
    while ((record = recordExchanger.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    Assert.assertTrue(capacity == counter);
}

Example 7 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class Consumer method test_seq.

// 测试SEQ
@Test
public void test_seq() {
    int capacity = 4;
    Record record = null;
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        this.channel.push(record);
    }
    for (int i = 0; i < capacity; i++) {
        record = this.channel.pull();
        System.out.println(record.getColumn(0).asLong());
        Assert.assertTrue(record.getColumn(0).asLong() == i);
    }
    List<Record> records = new ArrayList<Record>(capacity);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        records.add(record);
    }
    this.channel.pushAll(records);
    this.channel.pullAll(records);
    System.out.println(records.size());
    for (int i = 0; i < capacity; i++) {
        System.out.println(records.get(i).getColumn(0).asLong());
        Assert.assertTrue(records.get(i).getColumn(0).asLong() == i);
    }
}

Example 8 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class Consumer method test_Block.

@Test
public void test_Block() throws InterruptedException {
    int tryCount = 100;
    int capacity = ConfigurationProducer.produce().getInt(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY);
    System.out.println("capacity: " + capacity);
    Thread thread = new Thread(new Consumer(this.channel, tryCount * capacity));
    thread.start();
    List<Record> records = new ArrayList<Record>(capacity);
    for (int i = 0; i < capacity; i++) {
        Record record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        records.add(record);
    }
    for (int i = 0; i < tryCount; i++) {
        this.channel.pushAll(records);
    }
    Thread.sleep(5000L);
    List<Record> termindateRecords = new ArrayList<Record>();
    termindateRecords.add(TerminateRecord.get());
    this.channel.pushAll(termindateRecords);
    Thread.sleep(1000L);
    thread.join();
}

Example 9 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_单条超过buffer的脏数据.

@Test
public void test_BufferExchanger_单条超过buffer的脏数据() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    //测试单挑记录超过buffer大小
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 3);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    Channel channel2 = new MemoryChannel(configuration);
    channel2.setCommunication(new Communication());
    BufferedRecordExchanger recordExchanger2 = new BufferedRecordExchanger(channel2, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger2.sendToWriter(record);
    }
    ArgumentCaptor<Record> rgArg = ArgumentCaptor.forClass(Record.class);
    ArgumentCaptor<Exception> eArg = ArgumentCaptor.forClass(Exception.class);
    verify(pluginCollector, times(10)).collectDirtyRecord(rgArg.capture(), eArg.capture());
    recordExchanger2.flush();
    channel2.close();
    int counter = 0;
    while ((record = recordExchanger2.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(counter == 0);
}

Example 10 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_每条大小刚好是buffersize.

@Test
public void test_BufferExchanger_每条大小刚好是buffersize() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 229);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    final int capacity = 10;
    Record record = null;
    //测试单挑记录超过buffer大小
    Channel channel3 = new MemoryChannel(configuration);
    channel3.setCommunication(new Communication());
    final BufferedRecordExchanger recordExchangerWriter = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger recordExchangerReader = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger spy1 = spy(recordExchangerWriter);
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int counter = 0;
            Record record;
            while ((record = recordExchangerReader.getFromReader()) != null) {
                System.out.println(record.getColumn(0).toString());
                Assert.assertTrue(record.getColumn(0).asLong() == counter);
                counter++;
            }
            System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
            Assert.assertTrue(capacity == counter);
        }
    });
    t.start();
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        spy1.sendToWriter(record);
    }
    spy1.flush();
    channel3.close();
    t.join();
    verify(spy1, times(10)).flush();
}