Examples with Column com.alibaba.datax.common.element.Column used on opensource projects

Example 1 with Column

use of com.alibaba.datax.common.element.Column in project DataX by alibaba.

the class SubstrTransformer method evaluate.

@Override
public Record evaluate(Record record, Object... paras) {
    int columnIndex;
    int startIndex;
    int length;
    try {
        if (paras.length != 3) {
            throw new RuntimeException("dx_substr paras must be 3");
        }
        columnIndex = (Integer) paras[0];
        startIndex = Integer.valueOf((String) paras[1]);
        length = Integer.valueOf((String) paras[2]);
    } catch (Exception e) {
        throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "paras:" + Arrays.asList(paras).toString() + " => " + e.getMessage());
    }
    Column column = record.getColumn(columnIndex);
    try {
        String oriValue = column.asString();
        //如果字段为空，跳过subStr处理
        if (oriValue == null) {
            return record;
        }
        String newValue;
        if (startIndex > oriValue.length()) {
            throw new RuntimeException(String.format("dx_substr startIndex(%s) out of range(%s)", startIndex, oriValue.length()));
        }
        if (startIndex + length >= oriValue.length()) {
            newValue = oriValue.substring(startIndex, oriValue.length());
        } else {
            newValue = oriValue.substring(startIndex, startIndex + length);
        }
        record.setColumn(columnIndex, new StringColumn(newValue));
    } catch (Exception e) {
        throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e);
    }
    return record;
}

Example 2 with Column

use of com.alibaba.datax.common.element.Column in project DataX by alibaba.

the class PadTransformer method evaluate.

@Override
public Record evaluate(Record record, Object... paras) {
    int columnIndex;
    String padType;
    int length;
    String padString;
    try {
        if (paras.length != 4) {
            throw new RuntimeException("dx_pad paras must be 4");
        }
        columnIndex = (Integer) paras[0];
        padType = (String) paras[1];
        length = Integer.valueOf((String) paras[2]);
        padString = (String) paras[3];
    } catch (Exception e) {
        throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_ILLEGAL_PARAMETER, "paras:" + Arrays.asList(paras).toString() + " => " + e.getMessage());
    }
    Column column = record.getColumn(columnIndex);
    try {
        String oriValue = column.asString();
        //如果字段为空，作为空字符串处理
        if (oriValue == null) {
            oriValue = "";
        }
        String newValue;
        if (!padType.equalsIgnoreCase("r") && !padType.equalsIgnoreCase("l")) {
            throw new RuntimeException(String.format("dx_pad first para(%s) support l or r", padType));
        }
        if (length <= oriValue.length()) {
            newValue = oriValue.substring(0, length);
        } else {
            newValue = doPad(padType, oriValue, length, padString);
        }
        record.setColumn(columnIndex, new StringColumn(newValue));
    } catch (Exception e) {
        throw DataXException.asDataXException(TransformerErrorCode.TRANSFORMER_RUN_EXCEPTION, e.getMessage(), e);
    }
    return record;
}

Example 3 with Column

use of com.alibaba.datax.common.element.Column in project DataX by alibaba.

the class NormalTask method fetchLine.

@Override
public boolean fetchLine(Record record) throws Exception {
    Result result = super.getNextHbaseRow();
    if (null == result) {
        return false;
    }
    super.lastResult = result;
    try {
        byte[] hbaseColumnValue;
        String columnName;
        ColumnType columnType;
        byte[] columnFamily;
        byte[] qualifier;
        for (HbaseColumnCell cell : this.hbaseColumnCells) {
            columnType = cell.getColumnType();
            if (cell.isConstant()) {
                // 对常量字段的处理
                String constantValue = cell.getColumnValue();
                Column constantColumn = super.convertValueToAssignType(columnType, constantValue, cell.getDateformat());
                record.addColumn(constantColumn);
            } else {
                // 根据列名称获取值
                columnName = cell.getColumnName();
                if (Hbase094xHelper.isRowkeyColumn(columnName)) {
                    hbaseColumnValue = result.getRow();
                } else {
                    columnFamily = cell.getColumnFamily();
                    qualifier = cell.getQualifier();
                    hbaseColumnValue = result.getValue(columnFamily, qualifier);
                }
                Column hbaseColumn = super.convertBytesToAssignType(columnType, hbaseColumnValue, cell.getDateformat());
                record.addColumn(hbaseColumn);
            }
        }
    } catch (Exception e) {
        // 注意，这里catch的异常，期望是byte数组转换失败的情况。而实际上，string的byte数组，转成整数类型是不容易报错的。但是转成double类型容易报错。
        record.setColumn(0, new StringColumn(Bytes.toStringBinary(result.getRow())));
        throw e;
    }
    return true;
}

Example 4 with Column

use of com.alibaba.datax.common.element.Column in project DataX by alibaba.

the class NormalTask method fetchLine.

@Override
public boolean fetchLine(Record record) throws Exception {
    Result result = super.getNextHbaseRow();
    if (null == result) {
        return false;
    }
    super.lastResult = result;
    try {
        byte[] hbaseColumnValue;
        String columnName;
        ColumnType columnType;
        byte[] columnFamily;
        byte[] qualifier;
        for (HbaseColumnCell cell : this.hbaseColumnCells) {
            columnType = cell.getColumnType();
            if (cell.isConstant()) {
                // 对常量字段的处理
                String constantValue = cell.getColumnValue();
                Column constantColumn = super.convertValueToAssignType(columnType, constantValue, cell.getDateformat());
                record.addColumn(constantColumn);
            } else {
                // 根据列名称获取值
                columnName = cell.getColumnName();
                if (Hbase11xHelper.isRowkeyColumn(columnName)) {
                    hbaseColumnValue = result.getRow();
                } else {
                    columnFamily = cell.getColumnFamily();
                    qualifier = cell.getQualifier();
                    hbaseColumnValue = result.getValue(columnFamily, qualifier);
                }
                Column hbaseColumn = super.convertBytesToAssignType(columnType, hbaseColumnValue, cell.getDateformat());
                record.addColumn(hbaseColumn);
            }
        }
    } catch (Exception e) {
        // 注意，这里catch的异常，期望是byte数组转换失败的情况。而实际上，string的byte数组，转成整数类型是不容易报错的。但是转成double类型容易报错。
        record.setColumn(0, new StringColumn(Bytes.toStringBinary(result.getRow())));
        throw e;
    }
    return true;
}

Example 5 with Column

use of com.alibaba.datax.common.element.Column in project DataX by alibaba.

the class RecordExchangerTest method testMemeroySize.

@Test
public void testMemeroySize() throws Exception {
    Column longColumn = ColumnProducer.produceLongColumn(1);
    Column longColumn2 = new LongColumn("234567891");
    Column stringColumn = ColumnProducer.produceStringColumn("sringtest");
    Column boolColumn = ColumnProducer.produceBoolColumn(true);
    Column dateColumn = ColumnProducer.produceDateColumn(System.currentTimeMillis());
    Column bytesColumn = ColumnProducer.produceBytesColumn("test".getBytes("utf-8"));
    Assert.assertEquals(longColumn.getByteSize(), 8);
    Assert.assertEquals(longColumn2.getByteSize(), 9);
    Assert.assertEquals(stringColumn.getByteSize(), 9);
    Assert.assertEquals(boolColumn.getByteSize(), 1);
    Assert.assertEquals(dateColumn.getByteSize(), 8);
    Assert.assertEquals(bytesColumn.getByteSize(), 4);
    Record record = new DefaultRecord();
    record.addColumn(longColumn);
    record.addColumn(longColumn2);
    record.addColumn(stringColumn);
    record.addColumn(boolColumn);
    record.addColumn(dateColumn);
    record.addColumn(bytesColumn);
    Assert.assertEquals(record.getByteSize(), 39);
    // record classSize =  80
    // column classSize = 6*24
    Assert.assertEquals(record.getMemorySize(), 263);
}