Examples with BinarySortableSerDe org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe used on opensource projects

Example 1 with BinarySortableSerDe

use of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe in project hive by apache.

the class HBaseUtils method desierliazeDbNameTableNameFromPartitionKey.

private static List<String> desierliazeDbNameTableNameFromPartitionKey(byte[] key, Configuration conf) {
    StringBuffer names = new StringBuffer();
    names.append("dbName,tableName,");
    StringBuffer types = new StringBuffer();
    types.append("string,string,");
    BinarySortableSerDe serDe = new BinarySortableSerDe();
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
    try {
        serDe.initialize(conf, props);
        List deserializedkeys = ((List) serDe.deserialize(new BytesWritable(key))).subList(0, 2);
        List<String> keys = new ArrayList<>();
        for (int i = 0; i < deserializedkeys.size(); i++) {
            Object deserializedKey = deserializedkeys.get(i);
            if (deserializedKey == null) {
                throw new RuntimeException("Can't have a null dbname or tablename");
            } else {
                TypeInfo inputType = TypeInfoUtils.getTypeInfoFromTypeString("string");
                ObjectInspector inputOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
                Converter converter = ObjectInspectorConverters.getConverter(inputOI, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
                keys.add((String) converter.convert(deserializedKey));
            }
        }
        return keys;
    } catch (SerDeException e) {
        throw new RuntimeException("Error when deserialize key", e);
    }
}

Example 2 with BinarySortableSerDe

use of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe in project hive by apache.

the class PartitionKeyComparator method compareTo.

@Override
public int compareTo(byte[] value, int offset, int length) {
    byte[] bytes = Arrays.copyOfRange(value, offset, offset + length);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Get key " + new String(bytes));
    }
    BinarySortableSerDe serDe = new BinarySortableSerDe();
    List deserializedkeys = null;
    try {
        serDe.initialize(new Configuration(), serdeProps);
        deserializedkeys = ((List) serDe.deserialize(new BytesWritable(bytes))).subList(2, 2 + names.split(",").length);
    } catch (SerDeException e) {
        // don't bother with failed deserialization, continue with next key
        return 1;
    }
    for (int i = 0; i < ranges.size(); i++) {
        Range range = ranges.get(i);
        NativeRange nativeRange = nativeRanges.get(i);
        Comparable partVal = (Comparable) deserializedkeys.get(nativeRange.pos);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Try to match range " + partVal + ", start " + nativeRange.start + ", end " + nativeRange.end);
        }
        if (range.start == null || range.start.inclusive && partVal.compareTo(nativeRange.start) >= 0 || !range.start.inclusive && partVal.compareTo(nativeRange.start) > 0) {
            if (range.end == null || range.end.inclusive && partVal.compareTo(nativeRange.end) <= 0 || !range.end.inclusive && partVal.compareTo(nativeRange.end) < 0) {
                continue;
            }
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Fail to match range " + range.keyName + "-" + partVal + "[" + nativeRange.start + "," + nativeRange.end + "]");
        }
        return 1;
    }
    for (int i = 0; i < ops.size(); i++) {
        Operator op = ops.get(i);
        NativeOperator nativeOp = nativeOps.get(i);
        switch(op.type) {
            case LIKE:
                if (!deserializedkeys.get(nativeOp.pos).toString().matches(op.val)) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Fail to match operator " + op.keyName + "(" + deserializedkeys.get(nativeOp.pos) + ") LIKE " + nativeOp.val);
                    }
                    return 1;
                }
                break;
            case NOTEQUALS:
                if (nativeOp.val.equals(deserializedkeys.get(nativeOp.pos))) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Fail to match operator " + op.keyName + "(" + deserializedkeys.get(nativeOp.pos) + ")!=" + nativeOp.val);
                    }
                    return 1;
                }
                break;
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("All conditions satisfied:" + deserializedkeys);
    }
    return 0;
}

Example 3 with BinarySortableSerDe

use of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe in project SQLWindowing by hbutani.

the class WindowingKeySerializer method serialize.

/*
	 * copied from BinarySortableSerDe::serialize
	 */
static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
    // Is this field a null?
    if (o == null) {
        buffer.write((byte) 0, invert);
        return;
    }
    // This field is not a null.
    buffer.write((byte) 1, invert);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return;
                        }
                    case BOOLEAN:
                        {
                            boolean v = ((BooleanObjectInspector) poi).get(o);
                            buffer.write((byte) (v ? 2 : 1), invert);
                            return;
                        }
                    case BYTE:
                        {
                            ByteObjectInspector boi = (ByteObjectInspector) poi;
                            byte v = boi.get(o);
                            buffer.write((byte) (v ^ 0x80), invert);
                            return;
                        }
                    case SHORT:
                        {
                            ShortObjectInspector spoi = (ShortObjectInspector) poi;
                            short v = spoi.get(o);
                            buffer.write((byte) ((v >> 8) ^ 0x80), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case INT:
                        {
                            IntObjectInspector ioi = (IntObjectInspector) poi;
                            int v = ioi.get(o);
                            buffer.write((byte) ((v >> 24) ^ 0x80), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case LONG:
                        {
                            LongObjectInspector loi = (LongObjectInspector) poi;
                            long v = loi.get(o);
                            buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                            buffer.write((byte) (v >> 48), invert);
                            buffer.write((byte) (v >> 40), invert);
                            buffer.write((byte) (v >> 32), invert);
                            buffer.write((byte) (v >> 24), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case FLOAT:
                        {
                            FloatObjectInspector foi = (FloatObjectInspector) poi;
                            int v = Float.floatToIntBits(foi.get(o));
                            if ((v & (1 << 31)) != 0) {
                                // negative number, flip all bits
                                v = ~v;
                            } else {
                                // positive number, flip the first bit
                                v = v ^ (1 << 31);
                            }
                            buffer.write((byte) (v >> 24), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case DOUBLE:
                        {
                            DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                            long v = Double.doubleToLongBits(doi.get(o));
                            if ((v & (1L << 63)) != 0) {
                                // negative number, flip all bits
                                v = ~v;
                            } else {
                                // positive number, flip the first bit
                                v = v ^ (1L << 63);
                            }
                            buffer.write((byte) (v >> 56), invert);
                            buffer.write((byte) (v >> 48), invert);
                            buffer.write((byte) (v >> 40), invert);
                            buffer.write((byte) (v >> 32), invert);
                            buffer.write((byte) (v >> 24), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case STRING:
                        {
                            StringObjectInspector soi = (StringObjectInspector) poi;
                            Text t = soi.getPrimitiveWritableObject(o);
                            byte[] data = t.getBytes();
                            int length = t.getLength();
                            for (int i = 0; i < length; i++) {
                                if (data[i] == 0 || data[i] == 1) {
                                    buffer.write((byte) 1, invert);
                                    buffer.write((byte) (data[i] + 1), invert);
                                } else {
                                    buffer.write(data[i], invert);
                                }
                            }
                            buffer.write((byte) 0, invert);
                            return;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        default:
            {
                throw new RuntimeException("Unsupported type in WindowingKey : " + oi.getCategory());
            }
    }
}

Example 4 with BinarySortableSerDe

use of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe in project hive by apache.

the class HBaseUtils method deserializePartitionKey.

// Deserialize a partition key and return _only_ the partition values.
private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key, Configuration conf) {
    StringBuffer names = new StringBuffer();
    names.append("dbName,tableName,");
    StringBuffer types = new StringBuffer();
    types.append("string,string,");
    for (int i = 0; i < partitions.size(); i++) {
        names.append(partitions.get(i).getName());
        types.append(TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType()));
        if (i != partitions.size() - 1) {
            names.append(",");
            types.append(",");
        }
    }
    BinarySortableSerDe serDe = new BinarySortableSerDe();
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
    try {
        serDe.initialize(conf, props);
        List deserializedkeys = ((List) serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size() + 2);
        List<String> partitionKeys = new ArrayList<String>();
        for (int i = 0; i < deserializedkeys.size(); i++) {
            Object deserializedKey = deserializedkeys.get(i);
            if (deserializedKey == null) {
                partitionKeys.add(HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME));
            } else {
                TypeInfo inputType = TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType());
                ObjectInspector inputOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
                Converter converter = ObjectInspectorConverters.getConverter(inputOI, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
                partitionKeys.add((String) converter.convert(deserializedKey));
            }
        }
        return partitionKeys;
    } catch (SerDeException e) {
        throw new RuntimeException("Error when deserialize key", e);
    }
}

Example 5 with BinarySortableSerDe

use of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe in project hive by apache.

the class HybridHashTableContainer method setSerde.

@Override
public void setSerde(MapJoinObjectSerDeContext keyCtx, MapJoinObjectSerDeContext valCtx) throws SerDeException {
    AbstractSerDe keySerde = keyCtx.getSerDe(), valSerde = valCtx.getSerDe();
    if (writeHelper == null) {
        LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName());
        // We assume this hashtable is loaded only when tez is enabled
        LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector();
        writeHelper = new MapJoinBytesTableContainer.LazyBinaryKvWriter(keySerde, valSoi, valCtx.hasFilterTag());
        if (internalValueOi == null) {
            internalValueOi = valSoi;
        }
        if (sortableSortOrders == null) {
            sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders();
        }
        if (nullMarkers == null) {
            nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers();
        }
        if (notNullMarkers == null) {
            notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers();
        }
    }
}