Examples with LazyMap org.apache.hadoop.hive.serde2.lazy.LazyMap used on opensource projects

Example 6 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project presto by prestodb.

the class TestLazyMap method assertMapDecode.

public static void assertMapDecode(String encodedMap, Map<? extends Object, ? extends Object> expectedMap) {
    LazyMap lazyMap = (LazyMap) createLazyObject(getLazySimpleMapObjectInspector(LAZY_STRING_OBJECT_INSPECTOR, getLazyStringObjectInspector(false, (byte) 0), (byte) 2, (byte) 3, new Text("\\N"), false, (byte) 0));
    lazyMap.init(newByteArrayRef(encodedMap), 0, encodedMap.length());
    Map<Object, Object> map = lazyMap.getMap();
    assertEquals(map, expectedMap);
}

Example 7 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class AvroLazyObjectInspector method getStructFieldData.

@SuppressWarnings("unchecked")
@Override
public Object getStructFieldData(Object data, StructField f) {
    if (data == null) {
        return null;
    }
    int fieldID = f.getFieldID();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Getting struct field data for field: [" + f.getFieldName() + "] on data [" + data.getClass() + "]");
    }
    if (data instanceof LazyStruct) {
        LazyStruct row = (LazyStruct) data;
        // get the field out of struct
        Object rowField = row.getField(fieldID);
        if (rowField instanceof LazyStruct) {
            if (LOG.isDebugEnabled() && rowField != null) {
                LOG.debug("Deserializing struct [" + rowField.getClass() + "]");
            }
            return deserializeStruct(rowField, f.getFieldName());
        } else if (rowField instanceof LazyMap) {
            // We have found a map. Systematically deserialize the values of the map and return back the
            // map
            LazyMap lazyMap = (LazyMap) rowField;
            for (Entry<Object, Object> entry : lazyMap.getMap().entrySet()) {
                Object _key = entry.getKey();
                Object _value = entry.getValue();
                if (_value instanceof LazyStruct) {
                    lazyMap.getMap().put(_key, deserializeStruct(_value, f.getFieldName()));
                }
            }
            if (LOG.isDebugEnabled()) {
                LOG.debug("Returning a lazy map for field [" + f.getFieldName() + "]");
            }
            return lazyMap;
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Returning [" + rowField + "] for field [" + f.getFieldName() + "]");
            }
            // Just return the object. We need no further operation on it
            return rowField;
        }
    } else {
        // hive can operate on. Here we should be getting the same object back.
        if (!(data instanceof List)) {
            throw new IllegalArgumentException("data should be an instance of list");
        }
        if (!(fieldID < ((List<Object>) data).size())) {
            return null;
        }
        // lookup the field corresponding to the given field ID and return
        Object field = ((List<Object>) data).get(fieldID);
        if (field == null) {
            return null;
        }
        // convert to a lazy object and return
        return toLazyObject(field, f.getFieldObjectInspector());
    }
}

Example 8 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class TestHBaseSerDe method deserializeAndSerializeHiveMapHBaseColumnFamily.

private void deserializeAndSerializeHiveMapHBaseColumnFamily(HBaseSerDe hbaseSerDe, Result[] r, Put[] p, Object[][] expectedData, byte[][] rowKeys, byte[][] columnFamilies, byte[][][] columnQualifiersAndValues) throws SerDeException {
    StructObjectInspector soi = (StructObjectInspector) hbaseSerDe.getObjectInspector();
    List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
    assertEquals(8, fieldRefs.size());
    // Deserialize
    for (int i = 0; i < r.length; i++) {
        Object row = hbaseSerDe.deserialize(new ResultWritable(r[i]));
        Put serializedPut = ((PutWritable) hbaseSerDe.serialize(row, soi)).getPut();
        byte[] rowKey = serializedPut.getRow();
        for (int k = 0; k < rowKey.length; k++) {
            assertEquals(rowKey[k], rowKeys[i][k]);
        }
        assertEquals(columnFamilies.length, serializedPut.numFamilies());
        for (int j = 0; j < fieldRefs.size(); j++) {
            Object fieldData = soi.getStructFieldData(row, fieldRefs.get(j));
            assertNotNull(fieldData);
            if (fieldData instanceof LazyPrimitive<?, ?>) {
                assertEquals(expectedData[i][j], ((LazyPrimitive<?, ?>) fieldData).getWritableObject());
            } else if (fieldData instanceof LazyHBaseCellMap) {
                LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>) ((LazyHBaseCellMap) fieldData).getMapValueElement(expectedData[i][j]);
                assertEquals(expectedData[i][j], lazyPrimitive.getWritableObject());
            } else {
                fail("Error: field data not an instance of LazyPrimitive<?,?> or LazyMap");
            }
        }
    }
}

Example 9 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class TestLazyHBaseObject method testLazyHBaseCellMap1.

/**
   * Test the LazyMap class with Integer-to-String.
   * @throws SerDeException
   */
public void testLazyHBaseCellMap1() throws SerDeException {
    // Map of Integer to String
    Text nullSequence = new Text("\\N");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils.getTypeInfosFromTypeString("map<int,string>").get(0), new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    // Initialize a result
    List<KeyValue> kvs = new ArrayList<KeyValue>();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfa"), Bytes.toBytes("col1"), Bytes.toBytes("cfacol1")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfa"), Bytes.toBytes("col2"), Bytes.toBytes("cfacol2")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("2"), Bytes.toBytes("def")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("-1"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("0"), Bytes.toBytes("0")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("8"), Bytes.toBytes("abc")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfc"), Bytes.toBytes("col3"), Bytes.toBytes("cfccol3")));
    Result r = new Result(kvs);
    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(false);
    mapBinaryStorage.add(false);
    b.init(r, "cfb".getBytes(), mapBinaryStorage);
    assertEquals(new Text("def"), ((LazyString) b.getMapValueElement(new IntWritable(2))).getWritableObject());
    assertNull(b.getMapValueElement(new IntWritable(-1)));
    assertEquals(new Text("0"), ((LazyString) b.getMapValueElement(new IntWritable(0))).getWritableObject());
    assertEquals(new Text("abc"), ((LazyString) b.getMapValueElement(new IntWritable(8))).getWritableObject());
    assertNull(b.getMapValueElement(new IntWritable(12345)));
    assertEquals("{0:'0',2:'def',8:'abc'}".replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
}

Example 10 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project presto by prestodb.

the class RcFileTester method decodeRecordReaderValue.

private static Object decodeRecordReaderValue(Type type, Object actualValue) {
    if (actualValue instanceof LazyPrimitive) {
        actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    } else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    } else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    } else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    } else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    } else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    } else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    } else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    } else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    } else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    } else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    } else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = new SqlTimestamp((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
    } else if (actualValue instanceof StructObject) {
        StructObject structObject = (StructObject) actualValue;
        actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
    } else if (actualValue instanceof LazyBinaryArray) {
        actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
    } else if (actualValue instanceof LazyBinaryMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
    } else if (actualValue instanceof LazyArray) {
        actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
    } else if (actualValue instanceof LazyMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
    } else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    return actualValue;
}