Examples with LazySimpleSerDe org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe used on opensource projects

Example 31 with LazySimpleSerDe

use of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe in project hive by apache.

the class TestLazySimpleFast method testLazySimpleFastCase.

public void testLazySimpleFastCase(int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable {
    SerdeRandomRowSource source = new SerdeRandomRowSource();
    source.init(r, supportedTypes, depth);
    int rowCount = 100;
    Object[][] rows = source.randomRows(rowCount);
    if (doNonRandomFill) {
        MyTestClass.nonRandomRowFill(rows, source.primitiveCategories());
    }
    StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
    TypeInfo[] typeInfos = source.typeInfos();
    int columnCount = typeInfos.length;
    int writeColumnCount = columnCount;
    StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
    boolean doWriteFewerColumns = r.nextBoolean();
    if (doWriteFewerColumns) {
        writeColumnCount = 1 + r.nextInt(columnCount);
        if (writeColumnCount == columnCount) {
            doWriteFewerColumns = false;
        } else {
            writeRowStructObjectInspector = source.partialRowStructObjectInspector(writeColumnCount);
        }
    }
    String fieldNames = ObjectInspectorUtils.getFieldNames(rowStructObjectInspector);
    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowStructObjectInspector);
    // Use different separator values.
    byte[] separators = new byte[] { (byte) 9, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7, (byte) 8 };
    LazySimpleSerDe serde = getSerDe(fieldNames, fieldTypes);
    LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes, separators);
    LazySimpleSerDe serde_fewer = null;
    LazySerDeParameters serdeParams_fewer = null;
    if (doWriteFewerColumns) {
        String partialFieldNames = ObjectInspectorUtils.getFieldNames(writeRowStructObjectInspector);
        String partialFieldTypes = ObjectInspectorUtils.getFieldTypes(writeRowStructObjectInspector);
        serde_fewer = getSerDe(fieldNames, fieldTypes);
        serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes, separators);
    }
    testLazySimpleFast(source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */
    false, /* doWriteFewerColumns */
    false, r);
    testLazySimpleFast(source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */
    true, /* doWriteFewerColumns */
    false, r);
    if (doWriteFewerColumns) {
        testLazySimpleFast(source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */
        false, /* doWriteFewerColumns */
        true, r);
        testLazySimpleFast(source, rows, serde, rowStructObjectInspector, serde_fewer, writeRowStructObjectInspector, serdeParams, serdeParams_fewer, typeInfos, /* useIncludeColumns */
        true, /* doWriteFewerColumns */
        true, r);
    }
}

Example 32 with LazySimpleSerDe

use of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe in project hive by apache.

the class TestLazySimpleSerDe method testLazySimpleSerDeLastColumnTakesRest.

/**
 * Test the LazySimpleSerDe class with LastColumnTakesRest option.
 */
public void testLazySimpleSerDeLastColumnTakesRest() throws Throwable {
    try {
        // Create the SerDe
        LazySimpleSerDe serDe = new LazySimpleSerDe();
        Configuration conf = new Configuration();
        Properties tbl = createProperties();
        tbl.setProperty(serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, "true");
        SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
        // Data
        Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
        String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\ta\tb\t";
        Object[] expectedFieldsData = { new ByteWritable((byte) 123), new ShortWritable((short) 456), new IntWritable(789), new LongWritable(1000), new DoubleWritable(5.3), new Text("hive and hadoop"), new IntWritable(1), new Text("a\tb\t") };
        // Test
        deserializeAndSerialize(serDe, t, s, expectedFieldsData);
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 33 with LazySimpleSerDe

use of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe in project hive by apache.

the class TestLazySimpleSerDe method testLazySimpleSerDe.

/**
 * Test the LazySimpleSerDe class.
 */
public void testLazySimpleSerDe() throws Throwable {
    try {
        // Create the SerDe
        LazySimpleSerDe serDe = new LazySimpleSerDe();
        Configuration conf = new Configuration();
        Properties tbl = new Properties();
        tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
        tbl.setProperty("columns", "abyte,ashort,aint,along,adouble,astring,anullint,anullstring,aba");
        tbl.setProperty("columns.types", "tinyint:smallint:int:bigint:double:string:int:string:binary");
        tbl.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
        SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
        // Data
        Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\tNULL\t");
        t.append(new byte[] { (byte) Integer.parseInt("10111111", 2) }, 0, 1);
        StringBuilder sb = new StringBuilder("123\t456\t789\t1000\t5.3\thive and hadoop\t1\tNULL\t");
        String s = sb.append(new String(Base64.encodeBase64(new byte[] { (byte) Integer.parseInt("10111111", 2) }))).toString();
        Object[] expectedFieldsData = { new ByteWritable((byte) 123), new ShortWritable((short) 456), new IntWritable(789), new LongWritable(1000), new DoubleWritable(5.3), new Text("hive and hadoop"), new IntWritable(1), null, new BytesWritable(new byte[] { (byte) Integer.parseInt("10111111", 2) }) };
        // Test
        deserializeAndSerialize(serDe, t, s, expectedFieldsData);
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 34 with LazySimpleSerDe

use of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe in project hive by apache.

the class TestCrossMapEqualComparer method serializeAndDeserialize.

Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1, LazySimpleSerDe serde, LazySerDeParameters serdeParams) throws IOException, SerDeException {
    ByteStream.Output serializeStream = new ByteStream.Output();
    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams.getSeparators(), 0, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape());
    Text t = new Text(serializeStream.toByteArray());
    return serde.deserialize(t);
}

Example 35 with LazySimpleSerDe

use of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe in project hive by apache.

the class TestSimpleMapEqualComparer method serializeAndDeserialize.

Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1, LazySimpleSerDe serde, LazySerDeParameters serdeParams) throws IOException, SerDeException {
    ByteStream.Output serializeStream = new ByteStream.Output();
    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams.getSeparators(), 0, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape());
    Text t = new Text(serializeStream.toByteArray());
    return serde.deserialize(t);
}