Examples with StandardListObjectInspector org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector used on opensource projects

Example 1 with StandardListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector in project hive by apache.

the class MatchPath method createSelectListOI.

protected static StructObjectInspector createSelectListOI(MatchPath evaluator, PTFInputDef inpDef) {
    StructObjectInspector inOI = inpDef.getOutputShape().getOI();
    ArrayList<String> inputColumnNames = new ArrayList<String>();
    ArrayList<String> selectListNames = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (StructField f : inOI.getAllStructFieldRefs()) {
        String inputColName = evaluator.inputColumnNamesMap.get(f.getFieldName());
        if (inputColName != null) {
            inputColumnNames.add(inputColName);
            selectListNames.add(f.getFieldName());
            fieldOIs.add(f.getFieldObjectInspector());
        }
    }
    StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardStructObjectInspector(inputColumnNames, fieldOIs));
    ArrayList<ObjectInspector> selectFieldOIs = new ArrayList<ObjectInspector>();
    selectFieldOIs.addAll(fieldOIs);
    selectFieldOIs.add(pathAttrOI);
    selectListNames.add(MatchPath.PATHATTR_NAME);
    return ObjectInspectorFactory.getStandardStructObjectInspector(selectListNames, selectFieldOIs);
}

Example 2 with StandardListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector in project hive by apache.

the class TestAvroDeserializer method canDeserializeArrays.

@Test
public void canDeserializeArrays() throws SerDeException, IOException {
    Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE);
    GenericData.Record record = new GenericData.Record(s);
    List<String> list = new ArrayList<String>();
    list.add("Eccleston");
    list.add("Tennant");
    list.add("Smith");
    record.put("anArray", list);
    assertTrue(GENERIC_DATA.validate(s, record));
    System.out.println("Array-backed record = " + record);
    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
    AvroDeserializer de = new AvroDeserializer();
    ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
    assertEquals(1, row.size());
    Object theArrayObject = row.get(0);
    assertTrue(theArrayObject instanceof List);
    List theList = (List) theArrayObject;
    // Verify the raw object that's been created
    assertEquals("Eccleston", theList.get(0));
    assertEquals("Tennant", theList.get(1));
    assertEquals("Smith", theList.get(2));
    // Now go the correct way, through objectinspectors
    StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
    StructField fieldRefToArray = oi.getStructFieldRef("anArray");
    Object anArrayData = oi.getStructFieldData(row, fieldRefToArray);
    StandardListObjectInspector anArrayOI = (StandardListObjectInspector) fieldRefToArray.getFieldObjectInspector();
    assertEquals(3, anArrayOI.getListLength(anArrayData));
    JavaStringObjectInspector elementOI = (JavaStringObjectInspector) anArrayOI.getListElementObjectInspector();
    Object firstElement = anArrayOI.getListElement(anArrayData, 0);
    assertEquals("Eccleston", elementOI.getPrimitiveJavaObject(firstElement));
    assertTrue(firstElement instanceof String);
    Object secondElement = anArrayOI.getListElement(anArrayData, 1);
    assertEquals("Tennant", elementOI.getPrimitiveJavaObject(secondElement));
    assertTrue(secondElement instanceof String);
    Object thirdElement = anArrayOI.getListElement(anArrayData, 2);
    assertEquals("Smith", elementOI.getPrimitiveJavaObject(thirdElement));
    assertTrue(thirdElement instanceof String);
}

Example 3 with StandardListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector in project hive by apache.

the class StatsUtils method getSizeOfComplexTypes.

/**
   * Get the size of complex data types
   * @param conf
   *          - hive conf
   * @param oi
   *          - object inspector
   * @return raw data size
   */
public static long getSizeOfComplexTypes(HiveConf conf, ObjectInspector oi) {
    long result = 0;
    int length = 0;
    int listEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_LIST_NUM_ENTRIES);
    int mapEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAP_NUM_ENTRIES);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            String colTypeLowerCase = oi.getTypeName().toLowerCase();
            if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) {
                int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
                result += JavaDataModel.get().lengthForStringOfLength(avgColLen);
            } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) {
                int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
                result += JavaDataModel.get().lengthForByteArrayOfSize(avgColLen);
            } else {
                result += getAvgColLenOfFixedLengthTypes(colTypeLowerCase);
            }
            break;
        case LIST:
            if (oi instanceof StandardConstantListObjectInspector) {
                // constant list projection of known length
                StandardConstantListObjectInspector scloi = (StandardConstantListObjectInspector) oi;
                length = scloi.getWritableConstantValue().size();
                // check if list elements are primitive or Objects
                ObjectInspector leoi = scloi.getListElementObjectInspector();
                if (leoi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) {
                    result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length);
                } else {
                    result += JavaDataModel.get().lengthForObjectArrayOfSize(length);
                }
            } else {
                StandardListObjectInspector sloi = (StandardListObjectInspector) oi;
                // list overhead + (configured number of element in list * size of element)
                long elemSize = getSizeOfComplexTypes(conf, sloi.getListElementObjectInspector());
                result += JavaDataModel.get().arrayList() + (listEntries * elemSize);
            }
            break;
        case MAP:
            if (oi instanceof StandardConstantMapObjectInspector) {
                // constant map projection of known length
                StandardConstantMapObjectInspector scmoi = (StandardConstantMapObjectInspector) oi;
                result += getSizeOfMap(scmoi);
            } else {
                StandardMapObjectInspector smoi = (StandardMapObjectInspector) oi;
                result += getSizeOfComplexTypes(conf, smoi.getMapKeyObjectInspector());
                result += getSizeOfComplexTypes(conf, smoi.getMapValueObjectInspector());
                // hash map overhead
                result += JavaDataModel.get().hashMap(mapEntries);
            }
            break;
        case STRUCT:
            if (oi instanceof StandardConstantStructObjectInspector) {
                // constant map projection of known length
                StandardConstantStructObjectInspector scsoi = (StandardConstantStructObjectInspector) oi;
                result += getSizeOfStruct(scsoi);
            } else {
                StructObjectInspector soi = (StructObjectInspector) oi;
                // add constant object overhead for struct
                result += JavaDataModel.get().object();
                // add constant struct field names references overhead
                result += soi.getAllStructFieldRefs().size() * JavaDataModel.get().ref();
                for (StructField field : soi.getAllStructFieldRefs()) {
                    result += getSizeOfComplexTypes(conf, field.getFieldObjectInspector());
                }
            }
            break;
        case UNION:
            UnionObjectInspector uoi = (UnionObjectInspector) oi;
            // add constant object overhead for union
            result += JavaDataModel.get().object();
            // add constant size for unions tags
            result += uoi.getObjectInspectors().size() * JavaDataModel.get().primitive1();
            for (ObjectInspector foi : uoi.getObjectInspectors()) {
                result += getSizeOfComplexTypes(conf, foi);
            }
            break;
        default:
            break;
    }
    return result;
}

Example 4 with StandardListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector in project cdap by caskdata.

the class StandardObjectInspectorsTest method testPrimitiveTypesListObjectInspector.

@Test
public void testPrimitiveTypesListObjectInspector() throws Throwable {
    ObjectInspector oi;
    StandardListObjectInspector loi;
    // Byte array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Byte>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    byte[] bytes = new byte[] { 0, 1, 2 };
    Assert.assertEquals(3, loi.getListLength(bytes));
    Assert.assertEquals((byte) 0, loi.getListElement(bytes, 0));
    Assert.assertEquals((byte) 1, loi.getListElement(bytes, 1));
    Assert.assertEquals((byte) 2, loi.getListElement(bytes, 2));
    // Int array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Integer>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    int[] ints = new int[] { 0, 1, 2 };
    Assert.assertEquals(3, loi.getListLength(ints));
    Assert.assertEquals(0, loi.getListElement(ints, 0));
    Assert.assertEquals(1, loi.getListElement(ints, 1));
    Assert.assertEquals(2, loi.getListElement(ints, 2));
    // long array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Long>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    long[] longs = new long[] { 0, 1, 2 };
    Assert.assertEquals((long) 3, loi.getListLength(longs));
    Assert.assertEquals((long) 0, loi.getListElement(longs, 0));
    Assert.assertEquals((long) 1, loi.getListElement(longs, 1));
    Assert.assertEquals((long) 2, loi.getListElement(longs, 2));
    // double array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Double>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    double[] doubles = new double[] { 0.1d, 1.0d, 2.0d };
    Assert.assertEquals(3, loi.getListLength(doubles));
    Assert.assertEquals(0.1d, loi.getListElement(doubles, 0));
    Assert.assertEquals(1.0d, loi.getListElement(doubles, 1));
    Assert.assertEquals(2.0d, loi.getListElement(doubles, 2));
    // float array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Float>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    float[] floats = new float[] { 0.1f, 1.0f, 2.0f };
    Assert.assertEquals(3, loi.getListLength(floats));
    Assert.assertEquals(0.1f, loi.getListElement(floats, 0));
    Assert.assertEquals(1.0f, loi.getListElement(floats, 1));
    Assert.assertEquals(2.0f, loi.getListElement(floats, 2));
    // short array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Short>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    short[] shorts = new short[] { 0, 1, 2 };
    Assert.assertEquals(3, loi.getListLength(shorts));
    Assert.assertEquals((short) 0, loi.getListElement(shorts, 0));
    Assert.assertEquals((short) 1, loi.getListElement(shorts, 1));
    Assert.assertEquals((short) 2, loi.getListElement(shorts, 2));
    // short array
    oi = ObjectInspectorFactory.getReflectionObjectInspector(new TypeToken<List<Boolean>>() {
    }.getType());
    Assert.assertTrue(oi instanceof StandardListObjectInspector);
    loi = (StandardListObjectInspector) oi;
    boolean[] booleans = new boolean[] { true, false, false };
    Assert.assertEquals(3, loi.getListLength(booleans));
    Assert.assertEquals(true, loi.getListElement(booleans, 0));
    Assert.assertEquals(false, loi.getListElement(booleans, 1));
    Assert.assertEquals(false, loi.getListElement(booleans, 2));
}

Example 5 with StandardListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector in project cdap by caskdata.

the class StandardObjectInspectorsTest method testStandardListObjectInspector.

@Test
public void testStandardListObjectInspector() throws Throwable {
    try {
        StandardListObjectInspector loi1 = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        StandardListObjectInspector loi2 = ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        Assert.assertEquals(loi1, loi2);
        // metadata
        Assert.assertEquals(Category.LIST, loi1.getCategory());
        Assert.assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, loi1.getListElementObjectInspector());
        // null
        Assert.assertNull("loi1.getList(null) should be null.", loi1.getList(null));
        Assert.assertEquals("loi1.getListLength(null) should be -1.", loi1.getListLength(null), -1);
        Assert.assertNull("loi1.getListElement(null, 0) should be null", loi1.getListElement(null, 0));
        Assert.assertNull("loi1.getListElement(null, 100) should be null", loi1.getListElement(null, 100));
        // ArrayList
        ArrayList<Integer> list = new ArrayList<>();
        list.add(0);
        list.add(1);
        list.add(2);
        list.add(3);
        Assert.assertEquals(4, loi1.getList(list).size());
        Assert.assertEquals(4, loi1.getListLength(list));
        Assert.assertEquals(0, loi1.getListElement(list, 0));
        Assert.assertEquals(3, loi1.getListElement(list, 3));
        Assert.assertNull(loi1.getListElement(list, -1));
        Assert.assertNull(loi1.getListElement(list, 4));
        // Settable
        Object list4 = loi1.create(4);
        loi1.set(list4, 0, 0);
        loi1.set(list4, 1, 1);
        loi1.set(list4, 2, 2);
        loi1.set(list4, 3, 3);
        Assert.assertEquals(list, list4);
        loi1.resize(list4, 5);
        loi1.set(list4, 4, 4);
        loi1.resize(list4, 4);
        Assert.assertEquals(list, list4);
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}