Examples with LongColumn com.alibaba.datax.common.element.LongColumn used on opensource projects

Example 11 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger_不满32条到达buffer大小.

@Test
public void test_BufferExchanger_不满32条到达buffer大小() throws Exception {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    configuration.set(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY_BYTE, 500);
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    final int capacity = 10;
    Record record = null;
    //测试单挑记录超过buffer大小
    Channel channel3 = new MemoryChannel(configuration);
    channel3.setCommunication(new Communication());
    final BufferedRecordExchanger recordExchangerWriter = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger recordExchangerReader = new BufferedRecordExchanger(channel3, pluginCollector);
    final BufferedRecordExchanger spy1 = spy(recordExchangerWriter);
    Thread t = new Thread(new Runnable() {

        @Override
        public void run() {
            int counter = 0;
            Record record;
            while ((record = recordExchangerReader.getFromReader()) != null) {
                System.out.println(record.getColumn(0).toString());
                Assert.assertTrue(record.getColumn(0).asLong() == counter);
                counter++;
            }
            System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
            Assert.assertTrue(capacity == counter);
        }
    });
    t.start();
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        spy1.sendToWriter(record);
    }
    spy1.flush();
    channel3.close();
    t.join();
    verify(spy1, times(5)).flush();
}

Example 12 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class NormalTask method getVersion.

public long getVersion(Record record) {
    int index = versionColumn.getInt(Key.INDEX);
    long timestamp;
    if (index == -1) {
        //指定时间作为版本
        timestamp = versionColumn.getLong(Key.VALUE);
        if (timestamp < 0) {
            throw DataXException.asDataXException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, "您指定的版本非法!");
        }
    } else {
        //指定列作为版本,long/doubleColumn直接record.aslong, 其它类型尝试用yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss SSS去format
        if (index >= record.getColumnNumber()) {
            throw DataXException.asDataXException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, String.format("您的versionColumn配置项中中index值超出范围,根据reader端配置,index的值小于%s,而您配置的值为%s，请检查并修改.", record.getColumnNumber(), index));
        }
        if (record.getColumn(index).getRawData() == null) {
            throw DataXException.asDataXException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, "您指定的版本为空!");
        }
        SimpleDateFormat df_senconds = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        SimpleDateFormat df_ms = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS");
        if (record.getColumn(index) instanceof LongColumn || record.getColumn(index) instanceof DoubleColumn) {
            timestamp = record.getColumn(index).asLong();
        } else {
            Date date;
            try {
                date = df_ms.parse(record.getColumn(index).asString());
            } catch (ParseException e) {
                try {
                    date = df_senconds.parse(record.getColumn(index).asString());
                } catch (ParseException e1) {
                    LOG.info(String.format("您指定第[%s]列作为hbase写入版本,但在尝试用yyyy-MM-dd HH:mm:ss 和 yyyy-MM-dd HH:mm:ss SSS 去解析为Date时均出错,请检查并修改", index));
                    throw DataXException.asDataXException(Hbase11xWriterErrorCode.CONSTRUCT_VERSION_ERROR, e1);
                }
            }
            timestamp = date.getTime();
        }
    }
    return timestamp;
}