Examples with GroupFactory org.apache.parquet.example.data.GroupFactory used on opensource projects

Example 1 with GroupFactory

use of org.apache.parquet.example.data.GroupFactory in project parquet-mr by apache.

the class ValidatingColumnWriteStore method testRequiredOfRequired.

@Test
public void testRequiredOfRequired() {
    MessageType reqreqSchema = MessageTypeParser.parseMessageType("message Document {\n" + "  required group foo {\n" + "    required int64 bar;\n" + "  }\n" + "}\n");
    GroupFactory gf = new SimpleGroupFactory(reqreqSchema);
    Group g1 = gf.newGroup();
    g1.addGroup("foo").append("bar", 2l);
    testSchema(reqreqSchema, Arrays.asList(g1));
}

Example 2 with GroupFactory

use of org.apache.parquet.example.data.GroupFactory in project parquet-mr by apache.

the class ValidatingColumnWriteStore method testOptionalRequiredInteraction.

@Test
public void testOptionalRequiredInteraction() {
    for (int i = 0; i < 6; i++) {
        Type current = new PrimitiveType(Repetition.REQUIRED, PrimitiveTypeName.BINARY, "primitive");
        for (int j = 0; j < i; j++) {
            current = new GroupType(Repetition.REQUIRED, "req" + j, current);
        }
        MessageType groupSchema = new MessageType("schema" + i, current);
        GroupFactory gf = new SimpleGroupFactory(groupSchema);
        List<Group> groups = new ArrayList<Group>();
        Group root = gf.newGroup();
        Group currentGroup = root;
        for (int j = 0; j < i; j++) {
            currentGroup = currentGroup.addGroup(0);
        }
        currentGroup.add(0, Binary.fromString("foo"));
        groups.add(root);
        testSchema(groupSchema, groups);
    }
    for (int i = 0; i < 6; i++) {
        Type current = new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, "primitive");
        for (int j = 0; j < i; j++) {
            current = new GroupType(Repetition.REQUIRED, "req" + j, current);
        }
        MessageType groupSchema = new MessageType("schema" + (i + 6), current);
        GroupFactory gf = new SimpleGroupFactory(groupSchema);
        List<Group> groups = new ArrayList<Group>();
        Group rootDefined = gf.newGroup();
        Group rootUndefined = gf.newGroup();
        Group currentDefinedGroup = rootDefined;
        Group currentUndefinedGroup = rootUndefined;
        for (int j = 0; j < i; j++) {
            currentDefinedGroup = currentDefinedGroup.addGroup(0);
            currentUndefinedGroup = currentUndefinedGroup.addGroup(0);
        }
        currentDefinedGroup.add(0, Binary.fromString("foo"));
        groups.add(rootDefined);
        groups.add(rootUndefined);
        testSchema(groupSchema, groups);
    }
    for (int i = 0; i < 6; i++) {
        Type current = new PrimitiveType(Repetition.OPTIONAL, PrimitiveTypeName.BINARY, "primitive");
        for (int j = 0; j < 6; j++) {
            current = new GroupType(i == j ? Repetition.OPTIONAL : Repetition.REQUIRED, "req" + j, current);
        }
        MessageType groupSchema = new MessageType("schema" + (i + 12), current);
        GroupFactory gf = new SimpleGroupFactory(groupSchema);
        List<Group> groups = new ArrayList<Group>();
        Group rootDefined = gf.newGroup();
        Group rootUndefined = gf.newGroup();
        Group currentDefinedGroup = rootDefined;
        Group currentUndefinedGroup = rootUndefined;
        for (int j = 0; j < 6; j++) {
            currentDefinedGroup = currentDefinedGroup.addGroup(0);
            if (i < j) {
                currentUndefinedGroup = currentUndefinedGroup.addGroup(0);
            }
        }
        currentDefinedGroup.add(0, Binary.fromString("foo"));
        groups.add(rootDefined);
        groups.add(rootUndefined);
        testSchema(groupSchema, groups);
    }
}

Example 3 with GroupFactory

use of org.apache.parquet.example.data.GroupFactory in project parquet-mr by apache.

the class ValidatingColumnWriteStore method testOneOfEach.

@Test
public void testOneOfEach() {
    MessageType oneOfEachSchema = MessageTypeParser.parseMessageType(oneOfEach);
    GroupFactory gf = new SimpleGroupFactory(oneOfEachSchema);
    Group g1 = gf.newGroup().append("a", 1l).append("b", 2).append("c", 3.0f).append("d", 4.0d).append("e", true).append("f", Binary.fromString("6")).append("g", new NanoTime(1234, System.currentTimeMillis() * 1000)).append("h", Binary.fromString("abc"));
    testSchema(oneOfEachSchema, Arrays.asList(g1));
}

Example 4 with GroupFactory

use of org.apache.parquet.example.data.GroupFactory in project drill by axbaretto.

the class ParquetSimpleTestFileGenerator method main.

public static void main(String[] args) throws IOException {
    SimpleGroupFactory sgf = new SimpleGroupFactory(simpleSchema);
    GroupFactory gf = new SimpleGroupFactory(complexSchema);
    SimpleGroupFactory sngf = new SimpleGroupFactory(simpleNullableSchema);
    GroupFactory ngf = new SimpleGroupFactory(complexNullableSchema);
    ParquetWriter<Group> simpleWriter = initWriter(simpleSchema, "drill/parquet_test_file_simple");
    ParquetWriter<Group> complexWriter = initWriter(complexSchema, "drill/parquet_test_file_complex");
    ParquetWriter<Group> simpleNullableWriter = initWriter(simpleNullableSchema, "drill/parquet_test_file_simple_nullable");
    ParquetWriter<Group> complexNullableWriter = initWriter(complexNullableSchema, "drill/parquet_test_file_complex_nullable");
    ParquetSimpleTestFileGenerator.writeSimpleValues(sgf, simpleWriter, false);
    ParquetSimpleTestFileGenerator.writeSimpleValues(sngf, simpleNullableWriter, true);
    ParquetSimpleTestFileGenerator.writeComplexValues(gf, complexWriter, false);
    ParquetSimpleTestFileGenerator.writeComplexValues(ngf, complexNullableWriter, true);
    simpleWriter.close();
    complexWriter.close();
    simpleNullableWriter.close();
    complexNullableWriter.close();
}

Example 5 with GroupFactory

use of org.apache.parquet.example.data.GroupFactory in project drill by apache.

the class ParquetSimpleTestFileGenerator method main.

public static void main(String[] args) throws IOException {
    SimpleGroupFactory sgf = new SimpleGroupFactory(simpleSchema);
    GroupFactory gf = new SimpleGroupFactory(complexSchema);
    SimpleGroupFactory sngf = new SimpleGroupFactory(simpleNullableSchema);
    GroupFactory ngf = new SimpleGroupFactory(complexNullableSchema);
    // Generate files with dictionary encoding enabled and disabled
    ParquetWriter<Group> simpleWriter = initWriter(simpleSchema, "drill/parquet_test_file_simple", true);
    ParquetWriter<Group> complexWriter = initWriter(complexSchema, "drill/parquet_test_file_complex", true);
    ParquetWriter<Group> simpleNullableWriter = initWriter(simpleNullableSchema, "drill/parquet_test_file_simple_nullable", true);
    ParquetWriter<Group> complexNullableWriter = initWriter(complexNullableSchema, "drill/parquet_test_file_complex_nullable", true);
    ParquetWriter<Group> simpleNoDictWriter = initWriter(simpleSchema, "drill/parquet_test_file_simple_nodict", false);
    ParquetWriter<Group> complexNoDictWriter = initWriter(complexSchema, "drill/parquet_test_file_complex_nodict", false);
    ParquetWriter<Group> simpleNullableNoDictWriter = initWriter(simpleNullableSchema, "drill/parquet_test_file_simple_nullable_nodict", false);
    ParquetWriter<Group> complexNullableNoDictWriter = initWriter(complexNullableSchema, "drill/parquet_test_file_complex_nullable_nodict", false);
    ParquetSimpleTestFileGenerator.writeSimpleValues(sgf, simpleWriter, false);
    ParquetSimpleTestFileGenerator.writeSimpleValues(sngf, simpleNullableWriter, true);
    ParquetSimpleTestFileGenerator.writeComplexValues(gf, complexWriter, false);
    ParquetSimpleTestFileGenerator.writeComplexValues(ngf, complexNullableWriter, true);
    ParquetSimpleTestFileGenerator.writeSimpleValues(sgf, simpleNoDictWriter, false);
    ParquetSimpleTestFileGenerator.writeSimpleValues(sngf, simpleNullableNoDictWriter, true);
    ParquetSimpleTestFileGenerator.writeComplexValues(gf, complexNoDictWriter, false);
    ParquetSimpleTestFileGenerator.writeComplexValues(ngf, complexNullableNoDictWriter, true);
    simpleWriter.close();
    complexWriter.close();
    simpleNullableWriter.close();
    complexNullableWriter.close();
    simpleNoDictWriter.close();
    complexNoDictWriter.close();
    simpleNullableNoDictWriter.close();
    complexNullableNoDictWriter.close();
}