Examples with LazyFactory.createLazyObject org.apache.hadoop.hive.serde2.lazy.LazyFactory.createLazyObject used on opensource projects

Example 1 with LazyFactory.createLazyObject

use of org.apache.hadoop.hive.serde2.lazy.LazyFactory.createLazyObject in project hive by apache.

the class TestLazyBinarySerDe method testLazyBinaryObjectInspector.

/**
   * Test to see if byte[] with correct contents is generated by
   * LazyBinaryObjectInspector from input BytesWritable
   * @throws Throwable
   */
public void testLazyBinaryObjectInspector() throws Throwable {
    //create input ByteArrayRef
    ByteArrayRef inpBARef = new ByteArrayRef();
    inpBARef.setData(inpBArray);
    AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(TypeInfoFactory.binaryTypeInfo, false, (byte) 0);
    //create LazyBinary initialed with inputBA
    LazyBinary lazyBin = (LazyBinary) LazyFactory.createLazyObject(binInspector);
    lazyBin.init(inpBARef, 0, inpBArray.length);
    //use inspector to get a byte[] out of LazyBinary
    byte[] outBARef = (byte[]) binInspector.getPrimitiveJavaObject(lazyBin);
    assertTrue("compare input and output BAs", Arrays.equals(inpBArray, outBARef));
}

Example 2 with LazyFactory.createLazyObject

use of org.apache.hadoop.hive.serde2.lazy.LazyFactory.createLazyObject in project hive by apache.

the class AvroLazyObjectInspector method toLazyMapObject.

/**
   * Convert the given object to a lazy object using the given {@link ObjectInspector}
   *
   * @param obj Object to be converted to a {@link LazyObject}
   * @param oi ObjectInspector used for the conversion
   * @return the created {@link LazyObject lazy object}
   * */
@SuppressWarnings({ "rawtypes", "unchecked" })
private Object toLazyMapObject(Object obj, ObjectInspector objectInspector) {
    if (obj == null) {
        return null;
    }
    // avro guarantees that the key will be of type string. So we just need to worry about
    // deserializing the value here
    LazyMap lazyMap = (LazyMap) LazyFactory.createLazyObject(objectInspector);
    Map map = lazyMap.getMap();
    Map<Object, Object> origMap = (Map) obj;
    ObjectInspector keyObjectInspector = ((MapObjectInspector) objectInspector).getMapKeyObjectInspector();
    ObjectInspector valueObjectInspector = ((MapObjectInspector) objectInspector).getMapValueObjectInspector();
    for (Entry entry : origMap.entrySet()) {
        Object value = entry.getValue();
        map.put(toLazyPrimitiveObject(entry.getKey(), keyObjectInspector), toLazyObject(value, valueObjectInspector));
    }
    return lazyMap;
}

Example 3 with LazyFactory.createLazyObject

use of org.apache.hadoop.hive.serde2.lazy.LazyFactory.createLazyObject in project hive by apache.

the class TestLazyArrayMapStruct method testNestedinArrayAtLevel.

/**
   * @param nestingLevel
   * @param dtype
   * @param tableProp
   * @throws SerDeException
   */
private void testNestedinArrayAtLevel(int nestingLevel, ObjectInspector.Category dtype, Properties tableProp) throws SerDeException {
    //create type with nestingLevel levels of nesting
    //set inner schema for dtype
    String inSchema = null;
    switch(dtype) {
        case LIST:
            inSchema = "array<tinyint>";
            break;
        case MAP:
            inSchema = "map<string,int>";
            break;
        case STRUCT:
            inSchema = "struct<s:string,i:tinyint>";
            break;
        case UNION:
            inSchema = "uniontype<string,tinyint>";
            break;
        default:
            fail("type not supported by test case");
    }
    StringBuilder schema = new StringBuilder(inSchema);
    for (int i = 0; i < nestingLevel - 1; i++) {
        schema.insert(0, "array<");
        schema.append(">");
    }
    System.err.println("Testing nesting level " + nestingLevel + ". Using schema " + schema);
    // Create the SerDe
    LazySimpleSerDe serDe = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    tableProp.setProperty("columns", "narray");
    tableProp.setProperty("columns.types", schema.toString());
    SerDeUtils.initializeSerDe(serDe, conf, tableProp, null);
    LazySerDeParameters serdeParams = new LazySerDeParameters(conf, tableProp, LazySimpleSerDe.class.getName());
    //create the serialized string for type
    byte[] separators = serdeParams.getSeparators();
    System.err.println("Using separator " + (char) separators[nestingLevel]);
    byte[] serializedRow = null;
    switch(dtype) {
        case LIST:
            serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
            break;
        case MAP:
            byte kvSep = separators[nestingLevel + 1];
            byte kvPairSep = separators[nestingLevel];
            serializedRow = new byte[] { '1', kvSep, '1', kvPairSep, '2', kvSep, '2' };
            break;
        case STRUCT:
            serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
            break;
        case UNION:
            serializedRow = new byte[] { '0', separators[nestingLevel], '9' };
            break;
        default:
            fail("type not supported by test case");
    }
    //create LazyStruct with serialized string with expected separators
    StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
    LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi);
    TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length);
    //Get fields out of the lazy struct and check if they match expected
    // results
    //Get first level array
    LazyArray array = (LazyArray) struct.getField(0);
    //Peel off the n-1 levels to get to the underlying array
    for (int i = 0; i < nestingLevel - 2; i++) {
        array = (LazyArray) array.getListElementObject(0);
    }
    //verify the serialized format for dtype
    switch(dtype) {
        case LIST:
            LazyArray array1 = (LazyArray) array.getListElementObject(0);
            //check elements of the innermost array
            assertEquals(2, array1.getListLength());
            assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1.getListElementObject(0)).getWritableObject());
            assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1.getListElementObject(1)).getWritableObject());
            break;
        case MAP:
            LazyMap lazyMap = (LazyMap) array.getListElementObject(0);
            Map map = lazyMap.getMap();
            System.err.println(map);
            assertEquals(2, map.size());
            Iterator<Map.Entry<LazyString, LazyInteger>> it = map.entrySet().iterator();
            Entry<LazyString, LazyInteger> e1 = it.next();
            assertEquals(e1.getKey().getWritableObject(), new Text(new byte[] { '1' }));
            assertEquals(e1.getValue().getWritableObject(), new IntWritable(1));
            Entry<LazyString, LazyInteger> e2 = it.next();
            assertEquals(e2.getKey().getWritableObject(), new Text(new byte[] { '2' }));
            assertEquals(e2.getValue().getWritableObject(), new IntWritable(2));
            break;
        case STRUCT:
            LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0);
            //check elements of the innermost struct
            assertEquals(2, innerStruct.getFieldsAsList().size());
            assertEquals(new Text(new byte[] { '8' }), ((LazyString) innerStruct.getField(0)).getWritableObject());
            assertEquals(new ByteWritable((byte) 9), ((LazyByte) innerStruct.getField(1)).getWritableObject());
            break;
        case UNION:
            LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0);
            //check elements of the innermost union
            assertEquals(new Text(new byte[] { '9' }), ((LazyString) lazyUnion.getField()).getWritableObject());
            break;
        default:
            fail("type not supported by test case");
    }
    //test serialization
    Text serializedText = (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector());
    org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes());
}

Example 4 with LazyFactory.createLazyObject

use of org.apache.hadoop.hive.serde2.lazy.LazyFactory.createLazyObject in project hive by apache.

the class TestLazyArrayMapStruct method testLazyArray.

/**
   * Test the LazyArray class.
   */
public void testLazyArray() throws Throwable {
    try {
        // Array of Byte
        Text nullSequence = new Text("\\N");
        ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils.getTypeInfosFromTypeString("array<tinyint>").get(0), new byte[] { (byte) 1 }, 0, nullSequence, false, (byte) 0);
        LazyArray b = (LazyArray) LazyFactory.createLazyObject(oi);
        byte[] data = new byte[] { '-', '1', 1, '\\', 'N', 1, '8' };
        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
        assertNull(b.getListElementObject(-1));
        assertEquals(new ByteWritable((byte) -1), ((LazyByte) b.getListElementObject(0)).getWritableObject());
        assertEquals(new ByteWritable((byte) -1), ((LazyByte) b.getList().get(0)).getWritableObject());
        assertNull(b.getListElementObject(1));
        assertNull(b.getList().get(1));
        assertEquals(new ByteWritable((byte) 8), ((LazyByte) b.getListElementObject(2)).getWritableObject());
        assertEquals(new ByteWritable((byte) 8), ((LazyByte) b.getList().get(2)).getWritableObject());
        assertNull(b.getListElementObject(3));
        assertEquals(3, b.getList().size());
        // Array of String
        oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils.getTypeInfosFromTypeString("array<string>").get(0), new byte[] { (byte) '\t' }, 0, nullSequence, false, (byte) 0);
        b = (LazyArray) LazyFactory.createLazyObject(oi);
        data = new byte[] { 'a', 'b', '\t', 'c', '\t', '\\', 'N', '\t', '\t', 'd' };
        // Note: the first and last element of the byte[] are NOT used
        TestLazyPrimitive.initLazyObject(b, data, 1, data.length - 2);
        assertNull(b.getListElementObject(-1));
        assertEquals(new Text("b"), ((LazyString) b.getListElementObject(0)).getWritableObject());
        assertEquals(new Text("b"), ((LazyString) b.getList().get(0)).getWritableObject());
        assertEquals(new Text("c"), ((LazyString) b.getListElementObject(1)).getWritableObject());
        assertEquals(new Text("c"), ((LazyString) b.getList().get(1)).getWritableObject());
        assertNull((b.getListElementObject(2)));
        assertNull((b.getList().get(2)));
        assertEquals(new Text(""), ((LazyString) b.getListElementObject(3)).getWritableObject());
        assertEquals(new Text(""), ((LazyString) b.getList().get(3)).getWritableObject());
        assertEquals(new Text(""), ((LazyString) b.getListElementObject(4)).getWritableObject());
        assertEquals(new Text(""), ((LazyString) b.getList().get(4)).getWritableObject());
        assertNull((b.getListElementObject(5)));
        assertEquals(5, b.getList().size());
        // -- HIVE-4149
        b = (LazyArray) LazyFactory.createLazyObject(oi);
        data = new byte[] { 'a', '\t', '\\', 'N' };
        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
        assertEquals(new Text("a"), ((LazyString) b.getListElementObject(0)).getWritableObject());
        assertNull(b.getListElementObject(1));
        data = new byte[] { '\\', 'N', '\t', 'a' };
        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
        assertNull(b.getListElementObject(0));
        // twice (returns not cleaned cache)
        assertNull(b.getListElementObject(0));
        assertEquals(new Text("a"), ((LazyString) b.getListElementObject(1)).getWritableObject());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 5 with LazyFactory.createLazyObject

use of org.apache.hadoop.hive.serde2.lazy.LazyFactory.createLazyObject in project hive by apache.

the class TestAccumuloRowSerializer method testBinarySerialization.

@Test
public void testBinarySerialization() throws IOException, SerDeException {
    List<String> columns = Arrays.asList("row", "cq1", "cq2", "cq3");
    List<TypeInfo> types = Arrays.<TypeInfo>asList(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo, TypeInfoFactory.stringTypeInfo);
    List<String> typeNames = new ArrayList<String>(types.size());
    for (TypeInfo type : types) {
        typeNames.add(type.getTypeName());
    }
    Properties tableProperties = new Properties();
    tableProperties.setProperty(AccumuloSerDeParameters.COLUMN_MAPPINGS, ":rowid,cf:cq1#b,cf:cq2#b,cf:cq3");
    tableProperties.setProperty(serdeConstants.FIELD_DELIM, " ");
    tableProperties.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columns));
    tableProperties.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(',').join(typeNames));
    AccumuloSerDeParameters accumuloSerDeParams = new AccumuloSerDeParameters(new Configuration(), tableProperties, AccumuloSerDe.class.getSimpleName());
    LazySerDeParameters serDeParams = accumuloSerDeParams.getSerDeParameters();
    LazySimpleStructObjectInspector oi = (LazySimpleStructObjectInspector) LazyFactory.createLazyStructInspector(columns, types, serDeParams.getSeparators(), serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), serDeParams.getEscapeChar());
    AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams, accumuloSerDeParams.getColumnMappings(), new ColumnVisibility(), accumuloSerDeParams.getRowIdFactory());
    // Create the LazyStruct from the LazyStruct...Inspector
    LazyStruct obj = (LazyStruct) LazyFactory.createLazyObject(oi);
    ByteArrayRef byteRef = new ByteArrayRef();
    byteRef.setData(new byte[] { 'r', 'o', 'w', '1', ' ', '1', '0', ' ', '2', '0', ' ', 'v', 'a', 'l', 'u', 'e' });
    obj.init(byteRef, 0, byteRef.getData().length);
    Mutation m = (Mutation) serializer.serialize(obj, oi);
    Assert.assertArrayEquals("row1".getBytes(), m.getRow());
    List<ColumnUpdate> updates = m.getUpdates();
    Assert.assertEquals(3, updates.size());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    ColumnUpdate update = updates.get(0);
    Assert.assertEquals("cf", new String(update.getColumnFamily()));
    Assert.assertEquals("cq1", new String(update.getColumnQualifier()));
    out.writeInt(10);
    Assert.assertArrayEquals(baos.toByteArray(), update.getValue());
    update = updates.get(1);
    Assert.assertEquals("cf", new String(update.getColumnFamily()));
    Assert.assertEquals("cq2", new String(update.getColumnQualifier()));
    baos.reset();
    out.writeInt(20);
    Assert.assertArrayEquals(baos.toByteArray(), update.getValue());
    update = updates.get(2);
    Assert.assertEquals("cf", new String(update.getColumnFamily()));
    Assert.assertEquals("cq3", new String(update.getColumnQualifier()));
    Assert.assertEquals("value", new String(update.getValue()));
}