Examples with JavaStringObjectInspector org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector used on opensource projects

Example 1 with JavaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector in project presto by prestodb.

the class AbstractTestParquetReader method testSchemaWithOptionalOptionalRequiredFields.

@Test
public void testSchemaWithOptionalOptionalRequiredFields() throws Exception {
    MessageType parquetSchema = parseMessageType("message hive_schema {" + "  optional group a {" + "    optional group b {" + "      optional group c {" + "        required binary d (UTF8);" + "      }" + "    }" + "  }" + "} ");
    Type cType = RowType.from(singletonList(field("d", VARCHAR)));
    Type bType = RowType.from(singletonList(field("c", cType)));
    Type aType = RowType.from(singletonList(field("b", bType)));
    Iterable<String> dValues = asList("d1", "d2", "d3", "d4", "d5", "d6", "d7");
    Iterable<List> cValues = createNullableTestStructs(dValues);
    Iterable<List> bValues = createNullableTestStructs(cValues);
    List<List> aValues = createTestStructs(bValues);
    ObjectInspector cInspector = getStandardStructObjectInspector(singletonList("d"), singletonList(javaStringObjectInspector));
    ObjectInspector bInspector = getStandardStructObjectInspector(singletonList("c"), singletonList(cInspector));
    ObjectInspector aInspector = getStandardStructObjectInspector(singletonList("b"), singletonList(bInspector));
    tester.testRoundTrip(aInspector, aValues, aValues, "a", aType, Optional.of(parquetSchema));
}

Example 2 with JavaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector in project presto by prestodb.

the class AbstractTestParquetReader method testSingleLevelArrayOfStructOfStructOfSingleElement.

@Test
public void testSingleLevelArrayOfStructOfStructOfSingleElement() throws Exception {
    Iterable<List> structs = createTestStructs(transform(intsBetween(0, 31_234), Object::toString));
    Iterable<List> structsOfStructs = createTestStructs(structs);
    Iterable<List<List>> values = createTestArrays(structsOfStructs);
    List<String> structFieldNames = singletonList("test");
    List<String> structsOfStructsFieldNames = singletonList("test");
    Type structType = RowType.from(singletonList(field("test", VARCHAR)));
    Type structsOfStructsType = RowType.from(singletonList(field("test", structType)));
    ObjectInspector structObjectInspector = getStandardStructObjectInspector(structFieldNames, singletonList(javaStringObjectInspector));
    tester.testRoundTrip(getStandardListObjectInspector(getStandardStructObjectInspector(structsOfStructsFieldNames, singletonList(structObjectInspector))), values, values, new ArrayType(structsOfStructsType));
    tester.testSingleLevelArraySchemaRoundTrip(getStandardListObjectInspector(getStandardStructObjectInspector(structsOfStructsFieldNames, singletonList(structObjectInspector))), values, values, new ArrayType(structsOfStructsType));
}

Example 3 with JavaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector in project presto by prestodb.

the class AbstractTestParquetReader method testNestedMaps.

@Test
public void testNestedMaps() throws Exception {
    int nestingLevel = ThreadLocalRandom.current().nextInt(1, 15);
    Iterable<Integer> keys = intsBetween(0, 3_210);
    Iterable maps = limit(cycle(asList(null, "value2", "value3", null, null, "value6", "value7")), 3_210);
    ObjectInspector objectInspector = getStandardMapObjectInspector(javaIntObjectInspector, javaStringObjectInspector);
    Type type = mapType(INTEGER, VARCHAR);
    for (int i = 0; i < nestingLevel; i++) {
        maps = createNullableTestMaps(keys, maps);
        objectInspector = getStandardMapObjectInspector(javaIntObjectInspector, objectInspector);
        type = mapType(INTEGER, type);
    }
    maps = createTestMaps(keys, maps);
    tester.testRoundTrip(objectInspector, maps, maps, type);
}

Example 4 with JavaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector in project presto by prestodb.

the class AbstractTestParquetReader method testSchemaWithRequiredOptionalRequired2Fields.

@Test
public void testSchemaWithRequiredOptionalRequired2Fields() throws Exception {
    MessageType parquetSchema = parseMessageType("message hive_schema {" + "  optional group a {" + "    required group b {" + "      optional group c {" + "        required binary d (UTF8);" + "      }" + "    }" + "  }" + "  optional group e {" + "    required group f {" + "      optional group g {" + "        required binary h (UTF8);" + "      }" + "    }" + "  }" + "} ");
    Type cType = RowType.from(singletonList(field("d", VARCHAR)));
    Type bType = RowType.from(singletonList(field("c", cType)));
    Type aType = RowType.from(singletonList(field("b", bType)));
    Iterable<String> dValues = asList("d1", "d2", "d3", "d4", "d5", "d6", "d7");
    Iterable<List> cValues = createNullableTestStructs(dValues);
    List<List> bValues = createTestStructs(cValues);
    List<List> aValues = createTestStructs(bValues);
    Type gType = RowType.from(singletonList(field("h", VARCHAR)));
    Type fType = RowType.from(singletonList(field("g", gType)));
    Type eType = RowType.from(singletonList(field("f", fType)));
    Iterable<String> hValues = asList("h1", "h2", "h3", "h4", "h5", "h6", "h7");
    Iterable<List> gValues = createNullableTestStructs(hValues);
    List<List> fValues = createTestStructs(gValues);
    List<List> eValues = createTestStructs(fValues);
    ObjectInspector cInspector = getStandardStructObjectInspector(singletonList("d"), singletonList(javaStringObjectInspector));
    ObjectInspector bInspector = getStandardStructObjectInspector(singletonList("c"), singletonList(cInspector));
    ObjectInspector aInspector = getStandardStructObjectInspector(singletonList("b"), singletonList(bInspector));
    ObjectInspector gInspector = getStandardStructObjectInspector(singletonList("h"), singletonList(javaStringObjectInspector));
    ObjectInspector fInspector = getStandardStructObjectInspector(singletonList("g"), singletonList(gInspector));
    ObjectInspector eInspector = getStandardStructObjectInspector(singletonList("f"), singletonList(fInspector));
    tester.testRoundTrip(asList(aInspector, eInspector), new Iterable<?>[] { aValues, eValues }, new Iterable<?>[] { aValues, eValues }, asList("a", "e"), asList(aType, eType), Optional.of(parquetSchema), false);
}

Example 5 with JavaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector in project presto by prestodb.

the class AbstractTestParquetReader method testComplexNestedStructs.

@Test
public void testComplexNestedStructs() throws Exception {
    final int n = 30;
    Iterable<Integer> mapKeys = intsBetween(0, n);
    Iterable<Integer> intPrimitives = limit(cycle(asList(1, null, 3, null, 5, null, 7, null, null, null, 11, null, 13)), n);
    Iterable<String> stringPrimitives = limit(cycle(asList(null, "value2", "value3", null, null, "value6", "value7")), n);
    Iterable<Double> doublePrimitives = limit(cycle(asList(1.1, null, 3.3, null, 5.5, null, 7.7, null, null, null, 11.11, null, 13.13)), n);
    Iterable<Boolean> booleanPrimitives = limit(cycle(asList(null, true, false, null, null, true, false)), n);
    Iterable<String> mapStringKeys = Stream.generate(() -> UUID.randomUUID().toString()).limit(n).collect(Collectors.toList());
    Iterable<Map<Integer, String>> mapsIntString = createNullableTestMaps(mapKeys, stringPrimitives);
    Iterable<List<String>> arraysString = createNullableTestArrays(stringPrimitives);
    Iterable<Map<Integer, Double>> mapsIntDouble = createNullableTestMaps(mapKeys, doublePrimitives);
    Iterable<List<Boolean>> arraysBoolean = createNullableTestArrays(booleanPrimitives);
    Iterable<Map<String, String>> mapsStringString = createNullableTestMaps(mapStringKeys, stringPrimitives);
    List<String> struct1FieldNames = asList("mapIntStringField", "stringArrayField", "intField");
    Iterable<?> structs1 = createNullableTestStructs(mapsIntString, arraysString, intPrimitives);
    ObjectInspector struct1ObjectInspector = getStandardStructObjectInspector(struct1FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaStringObjectInspector), getStandardListObjectInspector(javaStringObjectInspector), javaIntObjectInspector));
    Type struct1Type = RowType.from(asList(field("mapIntStringField", mapType(INTEGER, VARCHAR)), field("stringArrayField", new ArrayType(VARCHAR)), field("intField", INTEGER)));
    List<String> struct2FieldNames = asList("mapIntStringField", "stringArrayField", "structField");
    Iterable<?> structs2 = createNullableTestStructs(mapsIntString, arraysString, structs1);
    ObjectInspector struct2ObjectInspector = getStandardStructObjectInspector(struct2FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaStringObjectInspector), getStandardListObjectInspector(javaStringObjectInspector), struct1ObjectInspector));
    Type struct2Type = RowType.from(asList(field("mapIntStringField", mapType(INTEGER, VARCHAR)), field("stringArrayField", new ArrayType(VARCHAR)), field("structField", struct1Type)));
    List<String> struct3FieldNames = asList("mapIntDoubleField", "booleanArrayField", "booleanField");
    Iterable<?> structs3 = createNullableTestStructs(mapsIntDouble, arraysBoolean, booleanPrimitives);
    ObjectInspector struct3ObjectInspector = getStandardStructObjectInspector(struct3FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaDoubleObjectInspector), getStandardListObjectInspector(javaBooleanObjectInspector), javaBooleanObjectInspector));
    Type struct3Type = RowType.from(asList(field("mapIntDoubleField", mapType(INTEGER, DOUBLE)), field("booleanArrayField", new ArrayType(BOOLEAN)), field("booleanField", BOOLEAN)));
    List<String> struct4FieldNames = asList("mapIntDoubleField", "booleanArrayField", "structField");
    Iterable<?> structs4 = createNullableTestStructs(mapsIntDouble, arraysBoolean, structs3);
    ObjectInspector struct4ObjectInspector = getStandardStructObjectInspector(struct4FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaDoubleObjectInspector), getStandardListObjectInspector(javaBooleanObjectInspector), struct3ObjectInspector));
    Type struct4Type = RowType.from(asList(field("mapIntDoubleField", mapType(INTEGER, DOUBLE)), field("booleanArrayField", new ArrayType(BOOLEAN)), field("structField", struct3Type)));
    List<String> structFieldNames = asList("structField1", "structField2", "structField3", "structField4", "mapIntDoubleField", "booleanArrayField", "mapStringStringField");
    List<ObjectInspector> objectInspectors = asList(struct1ObjectInspector, struct2ObjectInspector, struct3ObjectInspector, struct4ObjectInspector, getStandardMapObjectInspector(javaIntObjectInspector, javaDoubleObjectInspector), getStandardListObjectInspector(javaBooleanObjectInspector), getStandardMapObjectInspector(javaStringObjectInspector, javaStringObjectInspector));
    List<Type> types = ImmutableList.of(struct1Type, struct2Type, struct3Type, struct4Type, mapType(INTEGER, DOUBLE), new ArrayType(BOOLEAN), mapType(VARCHAR, VARCHAR));
    Iterable<?>[] values = new Iterable<?>[] { structs1, structs2, structs3, structs4, mapsIntDouble, arraysBoolean, mapsStringString };
    tester.assertRoundTrip(objectInspectors, values, values, structFieldNames, types, Optional.empty());
}