Examples with LongColumn com.alibaba.datax.common.element.LongColumn used on opensource projects

Example 1 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class MultiVersionTask method convertCellToLine.

private void convertCellToLine(KeyValue keyValue, Record record) throws Exception {
    byte[] rawRowkey = keyValue.getRow();
    long timestamp = keyValue.getTimestamp();
    byte[] cfAndQualifierName = Bytes.add(keyValue.getFamily(), MultiVersionTask.COLON_BYTE, keyValue.getQualifier());
    byte[] columnValue = keyValue.getValue();
    ColumnType rawRowkeyType = ColumnType.getByTypeName(familyQualifierMap.get(Constant.ROWKEY_FLAG).get(Key.TYPE));
    String familyQualifier = new String(cfAndQualifierName, Constant.DEFAULT_ENCODING);
    ColumnType columnValueType = ColumnType.getByTypeName(familyQualifierMap.get(familyQualifier).get(Key.TYPE));
    String columnValueFormat = familyQualifierMap.get(familyQualifier).get(Key.FORMAT);
    if (StringUtils.isBlank(columnValueFormat)) {
        columnValueFormat = Constant.DEFAULT_DATA_FORMAT;
    }
    record.addColumn(convertBytesToAssignType(rawRowkeyType, rawRowkey, columnValueFormat));
    record.addColumn(convertBytesToAssignType(ColumnType.STRING, cfAndQualifierName, columnValueFormat));
    // 直接忽略了用户配置的 timestamp 的类型
    record.addColumn(new LongColumn(timestamp));
    record.addColumn(convertBytesToAssignType(columnValueType, columnValue, columnValueFormat));
}

Example 2 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class NormalTask method getVersion.

public long getVersion(Record record) {
    int index = versionColumn.getInt(Key.INDEX);
    long timestamp;
    if (index == -1) {
        //指定时间作为版本
        timestamp = versionColumn.getLong(Key.VALUE);
        if (timestamp < 0) {
            throw DataXException.asDataXException(Hbase094xWriterErrorCode.CONSTRUCT_VERSION_ERROR, "您指定的版本非法!");
        }
    } else {
        //指定列作为版本,long/doubleColumn直接record.aslong, 其它类型尝试用yyyy-MM-dd HH:mm:ss,yyyy-MM-dd HH:mm:ss SSS去format
        if (index >= record.getColumnNumber()) {
            throw DataXException.asDataXException(Hbase094xWriterErrorCode.CONSTRUCT_VERSION_ERROR, String.format("您的versionColumn配置项中中index值超出范围,根据reader端配置,index的值小于%s,而您配置的值为%s，请检查并修改.", record.getColumnNumber(), index));
        }
        if (record.getColumn(index).getRawData() == null) {
            throw DataXException.asDataXException(Hbase094xWriterErrorCode.CONSTRUCT_VERSION_ERROR, "您指定的版本为空!");
        }
        SimpleDateFormat df_senconds = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        SimpleDateFormat df_ms = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS");
        if (record.getColumn(index) instanceof LongColumn || record.getColumn(index) instanceof DoubleColumn) {
            timestamp = record.getColumn(index).asLong();
        } else {
            Date date;
            try {
                date = df_ms.parse(record.getColumn(index).asString());
            } catch (ParseException e) {
                try {
                    date = df_senconds.parse(record.getColumn(index).asString());
                } catch (ParseException e1) {
                    LOG.info(String.format("您指定第[%s]列作为hbase写入版本,但在尝试用yyyy-MM-dd HH:mm:ss 和 yyyy-MM-dd HH:mm:ss SSS 去解析为Date时均出错,请检查并修改", index));
                    throw DataXException.asDataXException(Hbase094xWriterErrorCode.CONSTRUCT_VERSION_ERROR, e1);
                }
            }
            timestamp = date.getTime();
        }
    }
    return timestamp;
}

Example 3 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class Consumer method test_BlockAndSeq.

@Test
public void test_BlockAndSeq() throws InterruptedException {
    int tryCount = 100;
    int capacity = ConfigurationProducer.produce().getInt(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY);
    System.out.println("capacity: " + capacity);
    Thread thread = new Thread(new Consumer(this.channel, tryCount * capacity));
    thread.start();
    List<Record> records = new ArrayList<Record>(capacity);
    for (int i = 0; i < capacity; i++) {
        Record record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        records.add(record);
    }
    for (int i = 0; i < tryCount; i++) {
        this.channel.pushAll(records);
    }
    Thread.sleep(5000L);
    this.channel.push(TerminateRecord.get());
    Thread.sleep(1000L);
    thread.join();
}

Example 4 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger.

@Test
public void test_BufferExchanger() {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    Channel channel = new MemoryChannel(configuration);
    channel.setCommunication(new Communication());
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    BufferedRecordExchanger recordExchanger = new BufferedRecordExchanger(channel, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger.sendToWriter(record);
    }
    recordExchanger.flush();
    channel.close();
    int counter = 0;
    while ((record = recordExchanger.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(capacity == counter);
}

Example 5 with LongColumn

use of com.alibaba.datax.common.element.LongColumn in project DataX by alibaba.

the class RecordExchangerTest method testMemeroySize.

@Test
public void testMemeroySize() throws Exception {
    Column longColumn = ColumnProducer.produceLongColumn(1);
    Column longColumn2 = new LongColumn("234567891");
    Column stringColumn = ColumnProducer.produceStringColumn("sringtest");
    Column boolColumn = ColumnProducer.produceBoolColumn(true);
    Column dateColumn = ColumnProducer.produceDateColumn(System.currentTimeMillis());
    Column bytesColumn = ColumnProducer.produceBytesColumn("test".getBytes("utf-8"));
    Assert.assertEquals(longColumn.getByteSize(), 8);
    Assert.assertEquals(longColumn2.getByteSize(), 9);
    Assert.assertEquals(stringColumn.getByteSize(), 9);
    Assert.assertEquals(boolColumn.getByteSize(), 1);
    Assert.assertEquals(dateColumn.getByteSize(), 8);
    Assert.assertEquals(bytesColumn.getByteSize(), 4);
    Record record = new DefaultRecord();
    record.addColumn(longColumn);
    record.addColumn(longColumn2);
    record.addColumn(stringColumn);
    record.addColumn(boolColumn);
    record.addColumn(dateColumn);
    record.addColumn(bytesColumn);
    Assert.assertEquals(record.getByteSize(), 39);
    // record classSize =  80
    // column classSize = 6*24
    Assert.assertEquals(record.getMemorySize(), 263);
}