Examples with LongObjectInspector org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector used on opensource projects

Example 21 with LongObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector in project hive by apache.

the class ObjectInspectorUtils method hashCodeMurmur.

public static int hashCodeMurmur(Object o, ObjectInspector objIns, ByteBuffer byteBuffer) {
    if (o == null) {
        return 0;
    }
    // Reset the bytebuffer
    byteBuffer.clear();
    switch(objIns.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns);
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        return 0;
                    case BOOLEAN:
                        return (((BooleanObjectInspector) poi).get(o) ? 1 : 0);
                    case BYTE:
                        return ((ByteObjectInspector) poi).get(o);
                    case SHORT:
                        {
                            byteBuffer.putShort(((ShortObjectInspector) poi).get(o));
                            return Murmur3.hash32(byteBuffer.array(), 2);
                        }
                    case INT:
                        {
                            byteBuffer.putInt(((IntObjectInspector) poi).get(o));
                            return Murmur3.hash32(byteBuffer.array(), 4);
                        }
                    case LONG:
                        {
                            byteBuffer.putLong(((LongObjectInspector) poi).get(o));
                            return Murmur3.hash32(byteBuffer.array(), 8);
                        }
                    case FLOAT:
                        {
                            byteBuffer.putFloat(Float.floatToIntBits(((FloatObjectInspector) poi).get(o)));
                            return Murmur3.hash32(byteBuffer.array(), 4);
                        }
                    case DOUBLE:
                        {
                            // This hash function returns the same result as Double.hashCode()
                            // while DoubleWritable.hashCode returns a different result.
                            byteBuffer.putDouble(Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o)));
                            return Murmur3.hash32(byteBuffer.array(), 8);
                        }
                    case STRING:
                        {
                            // This hash function returns the same result as String.hashCode() when
                            // all characters are ASCII, while Text.hashCode() always returns a
                            // different result.
                            Text text = ((StringObjectInspector) poi).getPrimitiveWritableObject(o);
                            return Murmur3.hash32(text.getBytes(), text.getLength());
                        }
                    case CHAR:
                        {
                            Text text = ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).getStrippedValue();
                            return Murmur3.hash32(text.getBytes(), text.getLength());
                        }
                    case VARCHAR:
                        {
                            Text text = ((HiveVarcharObjectInspector) poi).getPrimitiveWritableObject(o).getTextValue();
                            return Murmur3.hash32(text.getBytes(), text.getLength());
                        }
                    case BINARY:
                        return Murmur3.hash32(((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).getBytes());
                    case DATE:
                        byteBuffer.putInt(((DateObjectInspector) poi).getPrimitiveWritableObject(o).getDays());
                        return Murmur3.hash32(byteBuffer.array(), 4);
                    case TIMESTAMP:
                        {
                            TimestampWritableV2 t = ((TimestampObjectInspector) poi).getPrimitiveWritableObject(o);
                            return Murmur3.hash32(t.getBytes());
                        }
                    case TIMESTAMPLOCALTZ:
                        return Murmur3.hash32((((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o)).getBytes());
                    case INTERVAL_YEAR_MONTH:
                        byteBuffer.putInt(((HiveIntervalYearMonthObjectInspector) poi).getPrimitiveWritableObject(o).hashCode());
                        return Murmur3.hash32(byteBuffer.array(), 4);
                    case INTERVAL_DAY_TIME:
                        byteBuffer.putInt(((HiveIntervalDayTimeObjectInspector) poi).getPrimitiveWritableObject(o).hashCode());
                        return Murmur3.hash32(byteBuffer.array(), 4);
                    case DECIMAL:
                        // compatible hash code.
                        return Murmur3.hash32(((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).getInternalStorage());
                    default:
                        {
                            throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                int r = 0;
                ListObjectInspector listOI = (ListObjectInspector) objIns;
                ObjectInspector elemOI = listOI.getListElementObjectInspector();
                for (int ii = 0; ii < listOI.getListLength(o); ++ii) {
                    // r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI);
                    r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer);
                }
                return r;
            }
        case MAP:
            {
                int r = 0;
                MapObjectInspector mapOI = (MapObjectInspector) objIns;
                ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
                ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
                Map<?, ?> map = mapOI.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^ hashCode(entry.getValue(), valueOI);
                }
                return r;
            }
        case STRUCT:
            int r = 0;
            StructObjectInspector structOI = (StructObjectInspector) objIns;
            List<? extends StructField> fields = structOI.getAllStructFieldRefs();
            for (StructField field : fields) {
                r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field), field.getFieldObjectInspector(), byteBuffer);
            }
            return r;
        case UNION:
            UnionObjectInspector uOI = (UnionObjectInspector) objIns;
            byte tag = uOI.getTag(o);
            return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag), byteBuffer);
        default:
            throw new RuntimeException("Unknown type: " + objIns.getTypeName());
    }
}

Example 22 with LongObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector in project SQLWindowing by hbutani.

the class WindowingKeySerializer method serialize.

/*
	 * copied from BinarySortableSerDe::serialize
	 */
static void serialize(OutputByteBuffer buffer, Object o, ObjectInspector oi, boolean invert) {
    // Is this field a null?
    if (o == null) {
        buffer.write((byte) 0, invert);
        return;
    }
    // This field is not a null.
    buffer.write((byte) 1, invert);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return;
                        }
                    case BOOLEAN:
                        {
                            boolean v = ((BooleanObjectInspector) poi).get(o);
                            buffer.write((byte) (v ? 2 : 1), invert);
                            return;
                        }
                    case BYTE:
                        {
                            ByteObjectInspector boi = (ByteObjectInspector) poi;
                            byte v = boi.get(o);
                            buffer.write((byte) (v ^ 0x80), invert);
                            return;
                        }
                    case SHORT:
                        {
                            ShortObjectInspector spoi = (ShortObjectInspector) poi;
                            short v = spoi.get(o);
                            buffer.write((byte) ((v >> 8) ^ 0x80), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case INT:
                        {
                            IntObjectInspector ioi = (IntObjectInspector) poi;
                            int v = ioi.get(o);
                            buffer.write((byte) ((v >> 24) ^ 0x80), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case LONG:
                        {
                            LongObjectInspector loi = (LongObjectInspector) poi;
                            long v = loi.get(o);
                            buffer.write((byte) ((v >> 56) ^ 0x80), invert);
                            buffer.write((byte) (v >> 48), invert);
                            buffer.write((byte) (v >> 40), invert);
                            buffer.write((byte) (v >> 32), invert);
                            buffer.write((byte) (v >> 24), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case FLOAT:
                        {
                            FloatObjectInspector foi = (FloatObjectInspector) poi;
                            int v = Float.floatToIntBits(foi.get(o));
                            if ((v & (1 << 31)) != 0) {
                                // negative number, flip all bits
                                v = ~v;
                            } else {
                                // positive number, flip the first bit
                                v = v ^ (1 << 31);
                            }
                            buffer.write((byte) (v >> 24), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case DOUBLE:
                        {
                            DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                            long v = Double.doubleToLongBits(doi.get(o));
                            if ((v & (1L << 63)) != 0) {
                                // negative number, flip all bits
                                v = ~v;
                            } else {
                                // positive number, flip the first bit
                                v = v ^ (1L << 63);
                            }
                            buffer.write((byte) (v >> 56), invert);
                            buffer.write((byte) (v >> 48), invert);
                            buffer.write((byte) (v >> 40), invert);
                            buffer.write((byte) (v >> 32), invert);
                            buffer.write((byte) (v >> 24), invert);
                            buffer.write((byte) (v >> 16), invert);
                            buffer.write((byte) (v >> 8), invert);
                            buffer.write((byte) v, invert);
                            return;
                        }
                    case STRING:
                        {
                            StringObjectInspector soi = (StringObjectInspector) poi;
                            Text t = soi.getPrimitiveWritableObject(o);
                            byte[] data = t.getBytes();
                            int length = t.getLength();
                            for (int i = 0; i < length; i++) {
                                if (data[i] == 0 || data[i] == 1) {
                                    buffer.write((byte) 1, invert);
                                    buffer.write((byte) (data[i] + 1), invert);
                                } else {
                                    buffer.write(data[i], invert);
                                }
                            }
                            buffer.write((byte) 0, invert);
                            return;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        default:
            {
                throw new RuntimeException("Unsupported type in WindowingKey : " + oi.getCategory());
            }
    }
}

Example 23 with LongObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector in project hive by apache.

the class BitmapObjectInput method readInt.

@Override
public int readInt() throws IOException {
    if (bufferIter.hasNext()) {
        LongObjectInspector loi = PrimitiveObjectInspectorFactory.writableLongObjectInspector;
        Long l = PrimitiveObjectInspectorUtils.getLong(bufferIter.next(), loi);
        return l.intValue();
    //return bufferIter.next().intValue();
    } else {
        throw new IOException();
    }
}

Example 24 with LongObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector in project hive by apache.

the class ColumnStatsTask method unpackDoubleStats.

private void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) {
    if (fName.equals("countnulls")) {
        long v = ((LongObjectInspector) oi).get(o);
        statsObj.getStatsData().getDoubleStats().setNumNulls(v);
    } else if (fName.equals("numdistinctvalues")) {
        long v = ((LongObjectInspector) oi).get(o);
        statsObj.getStatsData().getDoubleStats().setNumDVs(v);
    } else if (fName.equals("max")) {
        double d = ((DoubleObjectInspector) oi).get(o);
        statsObj.getStatsData().getDoubleStats().setHighValue(d);
    } else if (fName.equals("min")) {
        double d = ((DoubleObjectInspector) oi).get(o);
        statsObj.getStatsData().getDoubleStats().setLowValue(d);
    } else if (fName.equals("ndvbitvector")) {
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
        String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o);
        statsObj.getStatsData().getDoubleStats().setBitVectors(v);
        ;
    }
}

Example 25 with LongObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector in project hive by apache.

the class ColumnStatisticsObjTranslator method unpackDoubleStats.

private static void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException {
    if (fName.equals("countnulls")) {
        long v = ((LongObjectInspector) oi).get(o);
        statsObj.getStatsData().getDoubleStats().setNumNulls(v);
    } else if (fName.equals("numdistinctvalues")) {
        long v = ((LongObjectInspector) oi).get(o);
        statsObj.getStatsData().getDoubleStats().setNumDVs(v);
    } else if (fName.equals("max")) {
        double d = ((DoubleObjectInspector) oi).get(o);
        if (Double.isInfinite(d) || Double.isNaN(d)) {
            throw new UnsupportedDoubleException();
        }
        statsObj.getStatsData().getDoubleStats().setHighValue(d);
    } else if (fName.equals("min")) {
        double d = ((DoubleObjectInspector) oi).get(o);
        if (Double.isInfinite(d) || Double.isNaN(d)) {
            throw new UnsupportedDoubleException();
        }
        statsObj.getStatsData().getDoubleStats().setLowValue(d);
    } else if (fName.equals("ndvbitvector")) {
        PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
        byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
        statsObj.getStatsData().getDoubleStats().setBitVectors(buf);
        ;
    }
}