Examples with LazyMap org.apache.hadoop.hive.serde2.lazy.LazyMap used on opensource projects

Example 1 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class AvroLazyObjectInspector method toLazyMapObject.

/**
   * Convert the given object to a lazy object using the given {@link ObjectInspector}
   *
   * @param obj Object to be converted to a {@link LazyObject}
   * @param oi ObjectInspector used for the conversion
   * @return the created {@link LazyObject lazy object}
   * */
@SuppressWarnings({ "rawtypes", "unchecked" })
private Object toLazyMapObject(Object obj, ObjectInspector objectInspector) {
    if (obj == null) {
        return null;
    }
    // avro guarantees that the key will be of type string. So we just need to worry about
    // deserializing the value here
    LazyMap lazyMap = (LazyMap) LazyFactory.createLazyObject(objectInspector);
    Map map = lazyMap.getMap();
    Map<Object, Object> origMap = (Map) obj;
    ObjectInspector keyObjectInspector = ((MapObjectInspector) objectInspector).getMapKeyObjectInspector();
    ObjectInspector valueObjectInspector = ((MapObjectInspector) objectInspector).getMapValueObjectInspector();
    for (Entry entry : origMap.entrySet()) {
        Object value = entry.getValue();
        map.put(toLazyPrimitiveObject(entry.getKey(), keyObjectInspector), toLazyObject(value, valueObjectInspector));
    }
    return lazyMap;
}

Example 2 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class TestLazyArrayMapStruct method testNestedinArrayAtLevel.

/**
   * @param nestingLevel
   * @param dtype
   * @param tableProp
   * @throws SerDeException
   */
private void testNestedinArrayAtLevel(int nestingLevel, ObjectInspector.Category dtype, Properties tableProp) throws SerDeException {
    //create type with nestingLevel levels of nesting
    //set inner schema for dtype
    String inSchema = null;
    switch(dtype) {
        case LIST:
            inSchema = "array<tinyint>";
            break;
        case MAP:
            inSchema = "map<string,int>";
            break;
        case STRUCT:
            inSchema = "struct<s:string,i:tinyint>";
            break;
        case UNION:
            inSchema = "uniontype<string,tinyint>";
            break;
        default:
            fail("type not supported by test case");
    }
    StringBuilder schema = new StringBuilder(inSchema);
    for (int i = 0; i < nestingLevel - 1; i++) {
        schema.insert(0, "array<");
        schema.append(">");
    }
    System.err.println("Testing nesting level " + nestingLevel + ". Using schema " + schema);
    // Create the SerDe
    LazySimpleSerDe serDe = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    tableProp.setProperty("columns", "narray");
    tableProp.setProperty("columns.types", schema.toString());
    SerDeUtils.initializeSerDe(serDe, conf, tableProp, null);
    LazySerDeParameters serdeParams = new LazySerDeParameters(conf, tableProp, LazySimpleSerDe.class.getName());
    //create the serialized string for type
    byte[] separators = serdeParams.getSeparators();
    System.err.println("Using separator " + (char) separators[nestingLevel]);
    byte[] serializedRow = null;
    switch(dtype) {
        case LIST:
            serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
            break;
        case MAP:
            byte kvSep = separators[nestingLevel + 1];
            byte kvPairSep = separators[nestingLevel];
            serializedRow = new byte[] { '1', kvSep, '1', kvPairSep, '2', kvSep, '2' };
            break;
        case STRUCT:
            serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
            break;
        case UNION:
            serializedRow = new byte[] { '0', separators[nestingLevel], '9' };
            break;
        default:
            fail("type not supported by test case");
    }
    //create LazyStruct with serialized string with expected separators
    StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
    LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi);
    TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length);
    //Get fields out of the lazy struct and check if they match expected
    // results
    //Get first level array
    LazyArray array = (LazyArray) struct.getField(0);
    //Peel off the n-1 levels to get to the underlying array
    for (int i = 0; i < nestingLevel - 2; i++) {
        array = (LazyArray) array.getListElementObject(0);
    }
    //verify the serialized format for dtype
    switch(dtype) {
        case LIST:
            LazyArray array1 = (LazyArray) array.getListElementObject(0);
            //check elements of the innermost array
            assertEquals(2, array1.getListLength());
            assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1.getListElementObject(0)).getWritableObject());
            assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1.getListElementObject(1)).getWritableObject());
            break;
        case MAP:
            LazyMap lazyMap = (LazyMap) array.getListElementObject(0);
            Map map = lazyMap.getMap();
            System.err.println(map);
            assertEquals(2, map.size());
            Iterator<Map.Entry<LazyString, LazyInteger>> it = map.entrySet().iterator();
            Entry<LazyString, LazyInteger> e1 = it.next();
            assertEquals(e1.getKey().getWritableObject(), new Text(new byte[] { '1' }));
            assertEquals(e1.getValue().getWritableObject(), new IntWritable(1));
            Entry<LazyString, LazyInteger> e2 = it.next();
            assertEquals(e2.getKey().getWritableObject(), new Text(new byte[] { '2' }));
            assertEquals(e2.getValue().getWritableObject(), new IntWritable(2));
            break;
        case STRUCT:
            LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0);
            //check elements of the innermost struct
            assertEquals(2, innerStruct.getFieldsAsList().size());
            assertEquals(new Text(new byte[] { '8' }), ((LazyString) innerStruct.getField(0)).getWritableObject());
            assertEquals(new ByteWritable((byte) 9), ((LazyByte) innerStruct.getField(1)).getWritableObject());
            break;
        case UNION:
            LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0);
            //check elements of the innermost union
            assertEquals(new Text(new byte[] { '9' }), ((LazyString) lazyUnion.getField()).getWritableObject());
            break;
        default:
            fail("type not supported by test case");
    }
    //test serialization
    Text serializedText = (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector());
    org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes());
}

Example 3 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class TestAccumuloSerDe method testStructOfMapSerialization.

@Test
public void testStructOfMapSerialization() throws IOException, SerDeException {
    List<String> columns = Arrays.asList("row", "col");
    List<String> structColNames = Arrays.asList("map1", "map2");
    TypeInfo mapTypeInfo = TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo);
    // struct<map1:map<string,string>,map2:map<string,string>>,string
    List<TypeInfo> types = Arrays.<TypeInfo>asList(TypeInfoFactory.getStructTypeInfo(structColNames, Arrays.asList(mapTypeInfo, mapTypeInfo)), TypeInfoFactory.stringTypeInfo);
    Properties tableProperties = new Properties();
    tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, ":rowid,cf:cq");
    // Use the default separators [0, 1, 2, 3, ..., 7]
    tableProperties.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columns));
    tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(types));
    AccumuloSerDeParameters accumuloSerDeParams = new AccumuloSerDeParameters(new Configuration(), tableProperties, AccumuloSerDe.class.getSimpleName());
    LazySerDeParameters serDeParams = accumuloSerDeParams.getSerDeParameters();
    byte[] seps = serDeParams.getSeparators();
    // struct<map<k:v,k:v>_map<k:v,k:v>>>
    TypeInfo stringTypeInfo = TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
    LazyStringObjectInspector stringOI = (LazyStringObjectInspector) LazyFactory.createLazyObjectInspector(stringTypeInfo, new byte[] { 0 }, 0, serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar());
    LazyMapObjectInspector mapOI = LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(stringOI, stringOI, seps[3], seps[4], serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar());
    LazySimpleStructObjectInspector rowStructOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(structColNames, Arrays.<ObjectInspector>asList(mapOI, mapOI), (byte) seps[2], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), serDeParams.getEscapeChar());
    LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columns, Arrays.asList(rowStructOI, stringOI), seps[1], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), serDeParams.getEscapeChar());
    AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams, accumuloSerDeParams.getColumnMappings(), new ColumnVisibility(), accumuloSerDeParams.getRowIdFactory());
    Map<String, String> map1 = new HashMap<String, String>(), map2 = new HashMap<String, String>();
    map1.put("key10", "value10");
    map1.put("key11", "value11");
    map2.put("key20", "value20");
    map2.put("key21", "value21");
    ByteArrayRef byteRef = new ByteArrayRef();
    // Default separators are 1-indexed (instead of 0-indexed), thus the separator at offset 1 is
    // (byte) 2
    // The separator for the hive row is \x02, for the row Id struct, \x03, and the maps \x04 and
    // \x05
    String accumuloRow = "key10\5value10\4key11\5value11\3key20\5value20\4key21\5value21";
    LazyStruct entireStruct = (LazyStruct) LazyFactory.createLazyObject(structOI);
    byteRef.setData((accumuloRow + "\2foo").getBytes());
    entireStruct.init(byteRef, 0, byteRef.getData().length);
    Mutation m = serializer.serialize(entireStruct, structOI);
    Assert.assertArrayEquals(accumuloRow.getBytes(), m.getRow());
    Assert.assertEquals(1, m.getUpdates().size());
    ColumnUpdate update = m.getUpdates().get(0);
    Assert.assertEquals("cf", new String(update.getColumnFamily()));
    Assert.assertEquals("cq", new String(update.getColumnQualifier()));
    Assert.assertEquals("foo", new String(update.getValue()));
    AccumuloHiveRow haRow = new AccumuloHiveRow(new String(m.getRow()));
    haRow.add("cf", "cq", "foo".getBytes());
    LazyAccumuloRow lazyAccumuloRow = new LazyAccumuloRow(structOI);
    lazyAccumuloRow.init(haRow, accumuloSerDeParams.getColumnMappings(), accumuloSerDeParams.getRowIdFactory());
    List<Object> objects = lazyAccumuloRow.getFieldsAsList();
    Assert.assertEquals(2, objects.size());
    Assert.assertEquals("foo", objects.get(1).toString());
    LazyStruct rowStruct = (LazyStruct) objects.get(0);
    List<Object> rowObjects = rowStruct.getFieldsAsList();
    Assert.assertEquals(2, rowObjects.size());
    LazyMap rowMap = (LazyMap) rowObjects.get(0);
    Map<?, ?> actualMap = rowMap.getMap();
    System.out.println("Actual map 1: " + actualMap);
    Map<String, String> actualStringMap = new HashMap<String, String>();
    for (Entry<?, ?> entry : actualMap.entrySet()) {
        actualStringMap.put(entry.getKey().toString(), entry.getValue().toString());
    }
    Assert.assertEquals(map1, actualStringMap);
    rowMap = (LazyMap) rowObjects.get(1);
    actualMap = rowMap.getMap();
    System.out.println("Actual map 2: " + actualMap);
    actualStringMap = new HashMap<String, String>();
    for (Entry<?, ?> entry : actualMap.entrySet()) {
        actualStringMap.put(entry.getKey().toString(), entry.getValue().toString());
    }
    Assert.assertEquals(map2, actualStringMap);
}

Example 4 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class TestAccumuloSerDe method testMapSerialization.

@Test
public void testMapSerialization() throws Exception {
    Properties properties = new Properties();
    Configuration conf = new Configuration();
    properties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, ":rowID,cf:vals");
    properties.setProperty(serdeConstants.LIST_COLUMNS, "row,values");
    properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,map<string,string>");
    properties.setProperty(serdeConstants.COLLECTION_DELIM, ":");
    properties.setProperty(serdeConstants.MAPKEY_DELIM, "=");
    // Get one of the default separators to avoid having to set a custom separator
    char collectionSeparator = ':', kvSeparator = '=';
    serde.initialize(conf, properties);
    AccumuloHiveRow row = new AccumuloHiveRow();
    row.setRowId("r1");
    row.add("cf", "vals", ("k1" + kvSeparator + "v1" + collectionSeparator + "k2" + kvSeparator + "v2" + collectionSeparator + "k3" + kvSeparator + "v3").getBytes());
    Object obj = serde.deserialize(row);
    assertNotNull(obj);
    assertTrue(obj instanceof LazyAccumuloRow);
    LazyAccumuloRow lazyRow = (LazyAccumuloRow) obj;
    Object field0 = lazyRow.getField(0);
    assertNotNull(field0);
    assertTrue(field0 instanceof LazyString);
    assertEquals(row.getRowId(), ((LazyString) field0).getWritableObject().toString());
    Object field1 = lazyRow.getField(1);
    assertNotNull(field1);
    assertTrue(field1 instanceof LazyMap);
    LazyMap map = (LazyMap) field1;
    Map<Object, Object> untypedMap = map.getMap();
    assertEquals(3, map.getMapSize());
    Set<String> expectedKeys = new HashSet<String>();
    expectedKeys.add("k1");
    expectedKeys.add("k2");
    expectedKeys.add("k3");
    for (Entry<Object, Object> entry : untypedMap.entrySet()) {
        assertNotNull(entry.getKey());
        assertTrue(entry.getKey() instanceof LazyString);
        LazyString key = (LazyString) entry.getKey();
        assertNotNull(entry.getValue());
        assertTrue(entry.getValue() instanceof LazyString);
        LazyString value = (LazyString) entry.getValue();
        String strKey = key.getWritableObject().toString(), strValue = value.getWritableObject().toString();
        assertTrue(expectedKeys.remove(strKey));
        assertEquals(2, strValue.length());
        assertTrue(strValue.startsWith("v"));
        assertTrue(strValue.endsWith(Character.toString(strKey.charAt(1))));
    }
    assertTrue("Did not find expected keys: " + expectedKeys, expectedKeys.isEmpty());
}

Example 5 with LazyMap

use of org.apache.hadoop.hive.serde2.lazy.LazyMap in project hive by apache.

the class TestLazyHBaseObject method testLazyHBaseCellMap2.

/**
   * Test the LazyMap class with String-to-String.
   * @throws SerDeException
   */
public void testLazyHBaseCellMap2() throws SerDeException {
    // Map of String to String
    Text nullSequence = new Text("\\N");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0), new byte[] { (byte) '#', (byte) '\t' }, 0, nullSequence, false, (byte) 0);
    LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    // Initialize a result
    List<KeyValue> kvs = new ArrayList<KeyValue>();
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfa"), Bytes.toBytes("col1"), Bytes.toBytes("cfacol1")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfa"), Bytes.toBytes("col2"), Bytes.toBytes("cfacol2")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("2"), Bytes.toBytes("d\tf")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("-1"), Bytes.toBytes("")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("0"), Bytes.toBytes("0")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfb"), Bytes.toBytes("8"), Bytes.toBytes("abc")));
    kvs.add(new KeyValue(Bytes.toBytes("test-row"), Bytes.toBytes("cfc"), Bytes.toBytes("col3"), Bytes.toBytes("cfccol3")));
    Result r = new Result(kvs);
    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(false);
    mapBinaryStorage.add(false);
    b.init(r, "cfb".getBytes(), mapBinaryStorage);
    assertEquals(new Text("d\tf"), ((LazyString) b.getMapValueElement(new Text("2"))).getWritableObject());
    assertNull(b.getMapValueElement(new Text("-1")));
    assertEquals(new Text("0"), ((LazyString) b.getMapValueElement(new Text("0"))).getWritableObject());
    assertEquals(new Text("abc"), ((LazyString) b.getMapValueElement(new Text("8"))).getWritableObject());
    assertNull(b.getMapValueElement(new Text("-")));
    assertEquals("{'0':'0','2':'d\\tf','8':'abc'}".replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
}