Examples with LazyPrimitive org.apache.hadoop.hive.serde2.lazy.LazyPrimitive used on opensource projects

Example 11 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project hive by apache.

the class TestHBaseSerDe method deserializeAndSerializeHiveMapHBaseColumnFamily.

private void deserializeAndSerializeHiveMapHBaseColumnFamily(HBaseSerDe hbaseSerDe, Result[] r, Put[] p, Object[][] expectedData, byte[][] rowKeys, byte[][] columnFamilies, byte[][][] columnQualifiersAndValues) throws SerDeException {
    StructObjectInspector soi = (StructObjectInspector) hbaseSerDe.getObjectInspector();
    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
    assertEquals(8, fieldRefs.size());
    // Deserialize
    for (int i = 0; i < r.length; i++) {
        Object row = hbaseSerDe.deserialize(new ResultWritable(r[i]));
        Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut();
        byte[] rowKey = serializedPut.getRow();
        for (int k = 0; k < rowKey.length; k++) {
            assertEquals(rowKey[k], rowKeys[i][k]);
        }
        assertEquals(columnFamilies.length, serializedPut.numFamilies());
        for (int j = 0; j < fieldRefs.size(); j++) {
            Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
            assertNotNull(fieldData);
            if (fieldData instanceof LazyPrimitive<?, ?>) {
                assertEquals(expectedData[i][j], ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
            } else if (fieldData instanceof LazyHBaseCellMap) {
                LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>) ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedData[i][j]);
                assertEquals(expectedData[i][j], lazyPrimitive.getWritableObject());
            } else {
                fail("Error: field data not an instance of LazyPrimitive<?,?> or LazyMap");
            }
        }
    }
}

Example 12 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project hive by apache.

the class LazyAccumuloMap method parse.

protected void parse() {
    if (null == this.cachedMap) {
        this.cachedMap = new LinkedHashMap<Object, Object>();
    } else {
        this.cachedMap.clear();
    }
    LazyMapObjectInspector lazyMoi = getInspector();
    Text cf = new Text(columnMapping.getColumnFamily());
    for (ColumnTuple tuple : sourceRow.getTuples()) {
        String cq = tuple.getCq().toString();
        if (!cf.equals(tuple.getCf()) || !cq.startsWith(columnMapping.getColumnQualifierPrefix())) {
            // A column family or qualifier we don't want to include in the map
            continue;
        }
        // Because we append the cq prefix when serializing the column
        // we should also remove it when pulling it from Accumulo
        cq = cq.substring(columnMapping.getColumnQualifierPrefix().length());
        // Keys are always primitive, respect the binary
        LazyPrimitive<? extends ObjectInspector, ? extends Writable> key = LazyFactory.createLazyPrimitiveClass((PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(), ColumnEncoding.BINARY == columnMapping.getKeyEncoding());
        ByteArrayRef keyRef = new ByteArrayRef();
        keyRef.setData(cq.getBytes(Charsets.UTF_8));
        key.init(keyRef, 0, keyRef.getData().length);
        // Value can be anything, use the obj inspector and respect binary
        LazyObject<?> value = LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(), ColumnEncoding.BINARY == columnMapping.getValueEncoding());
        byte[] bytes = tuple.getValue();
        if (bytes == null || isNull(oi.getNullSequence(), bytes, 0, bytes.length)) {
            value.setNull();
        } else {
            ByteArrayRef valueRef = new ByteArrayRef();
            valueRef.setData(bytes);
            value.init(valueRef, 0, valueRef.getData().length);
        }
        cachedMap.put(key, value);
    }
    this.setParsed(true);
}

Example 13 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project hive by apache.

the class LazyAccumuloMap method getMapValueElement.

/**
   * Get the value in the map for the given key.
   *
   * @param key
   *          The key, a column qualifier, from the map
   * @return The object in the map at the given key
   */
@Override
public Object getMapValueElement(Object key) {
    if (!getParsed()) {
        parse();
    }
    for (Map.Entry<Object, Object> entry : cachedMap.entrySet()) {
        LazyPrimitive<?, ?> lazyKey = (LazyPrimitive<?, ?>) entry.getKey();
        // getWritableObject() will convert LazyPrimitive to actual primitive
        // writable objects.
        Object keyI = lazyKey.getWritableObject();
        if (keyI == null) {
            continue;
        }
        if (keyI.equals(key)) {
            // Got a match, return the value
            LazyObject<?> v = (LazyObject<?>) entry.getValue();
            return v == null ? v : v.getObject();
        }
    }
    return null;
}

Example 14 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project hive by apache.

the class TestHBaseSerDe method deserializeAndSerializeHiveMapHBaseColumnFamilyII.

private void deserializeAndSerializeHiveMapHBaseColumnFamilyII(HBaseSerDe hbaseSerDe, Result r, Put p, Object[] expectedData, byte[][] columnFamilies, byte[][] columnQualifiersAndValues) throws SerDeException {
    StructObjectInspector soi = (StructObjectInspector) hbaseSerDe.getObjectInspector();
    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
    assertEquals(9, fieldRefs.size());
    // Deserialize
    Object row = hbaseSerDe.deserialize(new ResultWritable(r));
    for (int j = 0; j < fieldRefs.size(); j++) {
        Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
        assertNotNull(fieldData);
        if (fieldData instanceof LazyPrimitive<?, ?>) {
            assertEquals(expectedData[j], ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
        } else if (fieldData instanceof LazyHBaseCellMap) {
            LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>) ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedData[j]);
            assertEquals(expectedData[j], lazyPrimitive.getWritableObject());
        } else {
            fail("Error: field data not an instance of LazyPrimitive<?, ?> or LazyHBaseCellMap");
        }
    }
    // Serialize
    Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut();
    assertEquals("Serialized data: ", p.toString(), serializedPut.toString());
}

Example 15 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project hive by apache.

the class TestLazyHBaseObject method testLazyHBaseCellMap3.

/**
   * Test the LazyHBaseCellMap class for the case where both the key and the value in the family
   * map are stored in binary format using the appropriate LazyPrimitive objects.
   * @throws SerDeException
   */
public void testLazyHBaseCellMap3() throws SerDeException {
    Text nullSequence = new Text("\\N");
    TypeInfo mapBinaryIntKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<int,int>");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(mapBinaryIntKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    LazyHBaseCellMap hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    List<KeyValue> kvs = new ArrayList<KeyValue>();
    byte[] rowKey = "row-key".getBytes();
    byte[] cfInt = "cf-int".getBytes();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(1), Bytes.toBytes(1)));
    Result result = new Result(kvs);
    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(true);
    mapBinaryStorage.add(true);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    IntWritable expectedIntValue = new IntWritable(1);
    LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(Integer.MIN_VALUE), Bytes.toBytes(Integer.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    expectedIntValue = new IntWritable(Integer.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(Integer.MAX_VALUE), Bytes.toBytes(Integer.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    expectedIntValue = new IntWritable(Integer.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryByteKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<tinyint,tinyint>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryByteKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfByte = "cf-byte".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte[] { (byte) 1 }, new byte[] { (byte) 1 }));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    ByteWritable expectedByteValue = new ByteWritable((byte) 1);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte[] { Byte.MIN_VALUE }, new byte[] { Byte.MIN_VALUE }));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    expectedByteValue = new ByteWritable(Byte.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte[] { Byte.MAX_VALUE }, new byte[] { Byte.MAX_VALUE }));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    expectedByteValue = new ByteWritable(Byte.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryShortKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<smallint,smallint>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryShortKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfShort = "cf-short".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes((short) 1), Bytes.toBytes((short) 1)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    ShortWritable expectedShortValue = new ShortWritable((short) 1);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MIN_VALUE), Bytes.toBytes(Short.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    expectedShortValue = new ShortWritable(Short.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MAX_VALUE), Bytes.toBytes(Short.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    expectedShortValue = new ShortWritable(Short.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryLongKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<bigint,bigint>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryLongKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfLong = "cf-long".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes((long) 1), Bytes.toBytes((long) 1)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    LongWritable expectedLongValue = new LongWritable(1);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MIN_VALUE), Bytes.toBytes(Long.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    expectedLongValue = new LongWritable(Long.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MAX_VALUE), Bytes.toBytes(Long.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    expectedLongValue = new LongWritable(Long.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryFloatKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<float,float>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryFloatKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfFloat = "cf-float".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) 1.0F), Bytes.toBytes((float) 1.0F)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    FloatWritable expectedFloatValue = new FloatWritable(1.0F);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MIN_VALUE), Bytes.toBytes((float) Float.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    expectedFloatValue = new FloatWritable(Float.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MAX_VALUE), Bytes.toBytes((float) Float.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    expectedFloatValue = new FloatWritable(Float.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryDoubleKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<double,double>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryDoubleKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfDouble = "cf-double".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(1.0), Bytes.toBytes(1.0)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    DoubleWritable expectedDoubleValue = new DoubleWritable(1.0);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MIN_VALUE), Bytes.toBytes(Double.MIN_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    expectedDoubleValue = new DoubleWritable(Double.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MAX_VALUE), Bytes.toBytes(Double.MAX_VALUE)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    expectedDoubleValue = new DoubleWritable(Double.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryBooleanKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<boolean,boolean>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryBooleanKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfBoolean = "cf-boolean".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(false), Bytes.toBytes(false)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
    BooleanWritable expectedBooleanValue = new BooleanWritable(false);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);
    assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(true), Bytes.toBytes(true)));
    result = new Result(kvs);
    hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
    expectedBooleanValue = new BooleanWritable(true);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);
    assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
}