Examples with LazyBinarySerDe org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe used on opensource projects

Example 6 with LazyBinarySerDe

use of org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe in project hive by apache.

the class LazyBinarySerDe method initialize.

/**
   * Initialize the SerDe with configuration and table information.
   */
@Override
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
    // Get column names and types
    String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
    String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
    String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    if (columnNameProperty.length() == 0) {
        columnNames = new ArrayList<String>();
    } else {
        columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
    }
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    assert (columnNames.size() == columnTypes.size());
    // Create row related objects
    rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    // Create the object inspector and the lazy binary struct object
    cachedObjectInspector = LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(rowTypeInfo);
    cachedLazyBinaryStruct = (LazyBinaryStruct) LazyBinaryFactory.createLazyBinaryObject(cachedObjectInspector);
    // output debug info
    LOG.debug("LazyBinarySerDe initialized with: columnNames=" + columnNames + " columnTypes=" + columnTypes);
    serializedSize = 0;
    stats = new SerDeStats();
    lastOperationSerialize = false;
    lastOperationDeserialize = false;
}

Example 7 with LazyBinarySerDe

use of org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe in project hive by apache.

the class TestStatsSerde method testLazyBinarySerDe.

/**
   * Test LazyBinarySerDe
   */
public void testLazyBinarySerDe() throws Throwable {
    try {
        System.out.println("test: testLazyBinarySerDe");
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        for (int i = 0; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        Properties schema = new Properties();
        schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
        schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
        LazyBinarySerDe serDe = new LazyBinarySerDe();
        SerDeUtils.initializeSerDe(serDe, new Configuration(), schema, null);
        deserializeAndSerializeLazyBinary(serDe, rows, rowOI);
        System.out.println("test: testLazyBinarySerDe - OK");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 8 with LazyBinarySerDe

use of org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe in project hive by apache.

the class TestLazyBinarySerDe method testLazyBinarySerDe.

/**
   * The test entrance function.
   *
   * @throws Throwable
   */
public void testLazyBinarySerDe() throws Throwable {
    try {
        System.out.println("Beginning Test TestLazyBinarySerDe:");
        // generate the data
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        for (int i = 0; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        // call the tests
        // 1/ test LazyBinarySerDe
        testLazyBinarySerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes));
        // 2/ test LazyBinaryMap
        testLazyBinaryMap(r);
        // 3/ test serialization and deserialization with different schemas
        testShorterSchemaDeserialization(r);
        // 4/ test serialization and deserialization with different schemas
        testLongerSchemaDeserialization(r);
        // 5/ test serialization and deserialization with different schemas
        testShorterSchemaDeserialization1(r);
        // 6/ test serialization and deserialization with different schemas
        testLongerSchemaDeserialization1(r);
        System.out.println("Test TestLazyBinarySerDe passed!");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 9 with LazyBinarySerDe

use of org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe in project hive by apache.

the class MapJoinBytesTableContainer method setSerde.

@Override
public void setSerde(MapJoinObjectSerDeContext keyContext, MapJoinObjectSerDeContext valueContext) throws SerDeException {
    AbstractSerDe keySerde = keyContext.getSerDe(), valSerde = valueContext.getSerDe();
    if (writeHelper == null) {
        LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName());
        if (keySerde instanceof BinarySortableSerDe && valSerde instanceof LazyBinarySerDe) {
            LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector();
            writeHelper = new LazyBinaryKvWriter(keySerde, valSoi, valueContext.hasFilterTag());
            internalValueOi = valSoi;
            sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders();
            nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers();
            notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers();
        } else {
            writeHelper = new KeyValueWriter(keySerde, valSerde, valueContext.hasFilterTag());
            internalValueOi = createInternalOi(valueContext);
            sortableSortOrders = null;
            nullMarkers = null;
            notNullMarkers = null;
        }
    }
}

Example 10 with LazyBinarySerDe

use of org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe in project hive by apache.

the class PTFTranslator method createLazyBinarySerDe.

/*
   * OI & Serde helper methods
   */
protected static AbstractSerDe createLazyBinarySerDe(Configuration cfg, StructObjectInspector oi, Map<String, String> serdePropsMap) throws SerDeException {
    serdePropsMap = serdePropsMap == null ? new LinkedHashMap<String, String>() : serdePropsMap;
    PTFDeserializer.addOIPropertiestoSerDePropsMap(oi, serdePropsMap);
    AbstractSerDe serDe = new LazyBinarySerDe();
    Properties p = new Properties();
    p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS));
    p.setProperty(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, serdePropsMap.get(org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES));
    SerDeUtils.initializeSerDe(serDe, cfg, p, null);
    return serDe;
}