Examples with ArrayType com.facebook.presto.common.type.ArrayType used on opensource projects

Example 21 with ArrayType

use of com.facebook.presto.common.type.ArrayType in project presto by prestodb.

the class AbstractTestParquetReader method testArraySchemas.

/**
 * Test reading various arrays schemas compatible with spec
 * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists
 */
@Test
public void testArraySchemas() throws Exception {
    MessageType parquetMrNullableSpecSchema = parseMessageType("message hive_schema {" + "  optional group my_list (LIST){" + "    repeated group list {" + "        required int32 element;" + "    }" + "  }" + "} ");
    Iterable<List<Integer>> nonNullArrayElements = createTestArrays(intsBetween(0, 31_234));
    tester.testRoundTrip(getStandardListObjectInspector(javaIntObjectInspector), nonNullArrayElements, nonNullArrayElements, "my_list", new ArrayType(INTEGER), Optional.of(parquetMrNullableSpecSchema));
    MessageType parquetMrNonNullSpecSchema = parseMessageType("message hive_schema {" + "  required group my_list (LIST){" + "    repeated group list {" + "        optional int32 element;" + "    }" + "  }" + "} ");
    Iterable<List<Integer>> values = createTestArrays(limit(cycle(asList(1, null, 3, 5, null, null, null, 7, 11, null, 13, 17)), 30_000));
    tester.assertRoundTrip(singletonList(getStandardListObjectInspector(javaIntObjectInspector)), new Iterable<?>[] { values }, new Iterable<?>[] { values }, singletonList("my_list"), singletonList(new ArrayType(INTEGER)), Optional.of(parquetMrNonNullSpecSchema));
    MessageType sparkSchema = parseMessageType("message hive_schema {" + "  optional group my_list (LIST){" + "    repeated group list {" + "        optional int32 element;" + "    }" + "  }" + "} ");
    tester.testRoundTrip(getStandardListObjectInspector(javaIntObjectInspector), values, values, "my_list", new ArrayType(INTEGER), Optional.of(sparkSchema));
    MessageType hiveSchema = parseMessageType("message hive_schema {" + "  optional group my_list (LIST){" + "    repeated group bag {" + "        optional int32 array_element;" + "    }" + "  }" + "} ");
    tester.testRoundTrip(getStandardListObjectInspector(javaIntObjectInspector), values, values, "my_list", new ArrayType(INTEGER), Optional.of(hiveSchema));
    MessageType customNamingSchema = parseMessageType("message hive_schema {" + "  optional group my_list (LIST){" + "    repeated group bag {" + "        optional int32 array;" + "    }" + "  }" + "} ");
    tester.testRoundTrip(getStandardListObjectInspector(javaIntObjectInspector), values, values, "my_list", new ArrayType(INTEGER), Optional.of(customNamingSchema));
}

Example 22 with ArrayType

use of com.facebook.presto.common.type.ArrayType in project presto by prestodb.

the class AbstractTestParquetReader method testArrayOfArrayOfStructOfArray.

@Test
public void testArrayOfArrayOfStructOfArray() throws Exception {
    Iterable<List<String>> stringArrayField = createNullableTestArrays(transform(intsBetween(0, 31_234), Object::toString));
    Iterable<List> structs = createNullableTestStructs(stringArrayField, limit(cycle(asList(1, null, 3, 5, null, 7, 11, null, 17)), 31_234));
    List<String> structFieldNames = asList("stringArrayField", "intField");
    Type structType = RowType.from(asList(field("stringArrayField", new ArrayType(VARCHAR)), field("intField", INTEGER)));
    Iterable<List<List>> arrays = createNullableTestArrays(structs);
    Iterable<List<List<List>>> values = createTestArrays(arrays);
    tester.testRoundTrip(getStandardListObjectInspector(getStandardListObjectInspector(getStandardStructObjectInspector(structFieldNames, asList(getStandardListObjectInspector(javaStringObjectInspector), javaIntObjectInspector)))), values, values, new ArrayType(new ArrayType(structType)));
}

Example 23 with ArrayType

use of com.facebook.presto.common.type.ArrayType in project presto by prestodb.

the class AbstractTestParquetReader method testComplexNestedStructs.

@Test
public void testComplexNestedStructs() throws Exception {
    final int n = 30;
    Iterable<Integer> mapKeys = intsBetween(0, n);
    Iterable<Integer> intPrimitives = limit(cycle(asList(1, null, 3, null, 5, null, 7, null, null, null, 11, null, 13)), n);
    Iterable<String> stringPrimitives = limit(cycle(asList(null, "value2", "value3", null, null, "value6", "value7")), n);
    Iterable<Double> doublePrimitives = limit(cycle(asList(1.1, null, 3.3, null, 5.5, null, 7.7, null, null, null, 11.11, null, 13.13)), n);
    Iterable<Boolean> booleanPrimitives = limit(cycle(asList(null, true, false, null, null, true, false)), n);
    Iterable<String> mapStringKeys = Stream.generate(() -> UUID.randomUUID().toString()).limit(n).collect(Collectors.toList());
    Iterable<Map<Integer, String>> mapsIntString = createNullableTestMaps(mapKeys, stringPrimitives);
    Iterable<List<String>> arraysString = createNullableTestArrays(stringPrimitives);
    Iterable<Map<Integer, Double>> mapsIntDouble = createNullableTestMaps(mapKeys, doublePrimitives);
    Iterable<List<Boolean>> arraysBoolean = createNullableTestArrays(booleanPrimitives);
    Iterable<Map<String, String>> mapsStringString = createNullableTestMaps(mapStringKeys, stringPrimitives);
    List<String> struct1FieldNames = asList("mapIntStringField", "stringArrayField", "intField");
    Iterable<?> structs1 = createNullableTestStructs(mapsIntString, arraysString, intPrimitives);
    ObjectInspector struct1ObjectInspector = getStandardStructObjectInspector(struct1FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaStringObjectInspector), getStandardListObjectInspector(javaStringObjectInspector), javaIntObjectInspector));
    Type struct1Type = RowType.from(asList(field("mapIntStringField", mapType(INTEGER, VARCHAR)), field("stringArrayField", new ArrayType(VARCHAR)), field("intField", INTEGER)));
    List<String> struct2FieldNames = asList("mapIntStringField", "stringArrayField", "structField");
    Iterable<?> structs2 = createNullableTestStructs(mapsIntString, arraysString, structs1);
    ObjectInspector struct2ObjectInspector = getStandardStructObjectInspector(struct2FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaStringObjectInspector), getStandardListObjectInspector(javaStringObjectInspector), struct1ObjectInspector));
    Type struct2Type = RowType.from(asList(field("mapIntStringField", mapType(INTEGER, VARCHAR)), field("stringArrayField", new ArrayType(VARCHAR)), field("structField", struct1Type)));
    List<String> struct3FieldNames = asList("mapIntDoubleField", "booleanArrayField", "booleanField");
    Iterable<?> structs3 = createNullableTestStructs(mapsIntDouble, arraysBoolean, booleanPrimitives);
    ObjectInspector struct3ObjectInspector = getStandardStructObjectInspector(struct3FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaDoubleObjectInspector), getStandardListObjectInspector(javaBooleanObjectInspector), javaBooleanObjectInspector));
    Type struct3Type = RowType.from(asList(field("mapIntDoubleField", mapType(INTEGER, DOUBLE)), field("booleanArrayField", new ArrayType(BOOLEAN)), field("booleanField", BOOLEAN)));
    List<String> struct4FieldNames = asList("mapIntDoubleField", "booleanArrayField", "structField");
    Iterable<?> structs4 = createNullableTestStructs(mapsIntDouble, arraysBoolean, structs3);
    ObjectInspector struct4ObjectInspector = getStandardStructObjectInspector(struct4FieldNames, asList(getStandardMapObjectInspector(javaIntObjectInspector, javaDoubleObjectInspector), getStandardListObjectInspector(javaBooleanObjectInspector), struct3ObjectInspector));
    Type struct4Type = RowType.from(asList(field("mapIntDoubleField", mapType(INTEGER, DOUBLE)), field("booleanArrayField", new ArrayType(BOOLEAN)), field("structField", struct3Type)));
    List<String> structFieldNames = asList("structField1", "structField2", "structField3", "structField4", "mapIntDoubleField", "booleanArrayField", "mapStringStringField");
    List<ObjectInspector> objectInspectors = asList(struct1ObjectInspector, struct2ObjectInspector, struct3ObjectInspector, struct4ObjectInspector, getStandardMapObjectInspector(javaIntObjectInspector, javaDoubleObjectInspector), getStandardListObjectInspector(javaBooleanObjectInspector), getStandardMapObjectInspector(javaStringObjectInspector, javaStringObjectInspector));
    List<Type> types = ImmutableList.of(struct1Type, struct2Type, struct3Type, struct4Type, mapType(INTEGER, DOUBLE), new ArrayType(BOOLEAN), mapType(VARCHAR, VARCHAR));
    Iterable<?>[] values = new Iterable<?>[] { structs1, structs2, structs3, structs4, mapsIntDouble, arraysBoolean, mapsStringString };
    tester.assertRoundTrip(objectInspectors, values, values, structFieldNames, types, Optional.empty());
}

Example 24 with ArrayType

use of com.facebook.presto.common.type.ArrayType in project presto by prestodb.

the class AbstractTestParquetReader method testArrayMaxReadBytes.

@Test
public void testArrayMaxReadBytes() throws Exception {
    DataSize maxReadBlockSize = new DataSize(1_000, DataSize.Unit.BYTE);
    Iterable<List<Integer>> values = createFixedTestArrays(limit(cycle(asList(1, null, 3, 5, null, null, null, 7, 11, null, 13, 17)), 30_000));
    tester.testMaxReadBytes(getStandardListObjectInspector(javaIntObjectInspector), values, values, new ArrayType(INTEGER), maxReadBlockSize);
}

Example 25 with ArrayType

use of com.facebook.presto.common.type.ArrayType in project presto by prestodb.

the class AbstractTestParquetReader method testStructOfNullableArrayBetweenNonNullFields.

@Test
public void testStructOfNullableArrayBetweenNonNullFields() throws Exception {
    Iterable<Integer> intPrimitives = intsBetween(0, 10_000);
    Iterable<String> stringPrimitives = limit(cycle(asList(null, "value2", "value3", null, null, "value6", "value7")), 10_000);
    Iterable<List<String>> stringArrayField = createNullableTestArrays(stringPrimitives);
    List<List> values = createTestStructs(intPrimitives, stringArrayField, intPrimitives);
    List<String> structFieldNames = asList("intField1", "arrayStringField", "intField2");
    Type structType = RowType.from(asList(field("intField1", INTEGER), field("arrayStringField", new ArrayType(VARCHAR)), field("intField2", INTEGER)));
    tester.testRoundTrip(getStandardStructObjectInspector(structFieldNames, asList(javaIntObjectInspector, getStandardListObjectInspector(javaStringObjectInspector), javaIntObjectInspector)), values, values, structType);
}