Examples with GenericRecord org.apache.iceberg.data.GenericRecord used on opensource projects

Example 1 with GenericRecord

use of org.apache.iceberg.data.GenericRecord in project hive by apache.

the class TestHiveIcebergStorageHandlerLocalScan method testArrayOfStructsInTable.

@Test
public void testArrayOfStructsInTable() throws IOException {
    Schema schema = new Schema(required(1, "arrayofstructs", Types.ListType.ofRequired(2, Types.StructType.of(required(3, "something", Types.DoubleType.get()), required(4, "someone", Types.LongType.get()), required(5, "somewhere", Types.StringType.get())))));
    List<Record> records = testTables.createTableWithGeneratedRecords(shell, "arraytable", schema, fileFormat, 1);
    // access an element from a struct in an array
    for (int i = 0; i < records.size(); i++) {
        List<?> expectedList = (List<?>) records.get(i).getField("arrayofstructs");
        for (int j = 0; j < expectedList.size(); j++) {
            List<Object[]> queryResult = shell.executeStatement(String.format("SELECT arrayofstructs[%d].something, " + "arrayofstructs[%d].someone, arrayofstructs[%d].somewhere FROM default.arraytable LIMIT 1 " + "OFFSET %d", j, j, j, i));
            GenericRecord genericRecord = (GenericRecord) expectedList.get(j);
            Assert.assertEquals(genericRecord.getField("something"), queryResult.get(0)[0]);
            Assert.assertEquals(genericRecord.getField("someone"), queryResult.get(0)[1]);
            Assert.assertEquals(genericRecord.getField("somewhere"), queryResult.get(0)[2]);
        }
    }
}

Example 2 with GenericRecord

use of org.apache.iceberg.data.GenericRecord in project hive by apache.

the class TestHiveIcebergTypes method testStructOfPrimitivesInTable.

@Test
public void testStructOfPrimitivesInTable() throws IOException {
    Schema schema = new Schema(required(1, "structofprimitives", Types.StructType.of(required(2, "key", Types.StringType.get()), required(3, "value", Types.IntegerType.get()))));
    List<Record> records = testTables.createTableWithGeneratedRecords(shell, "structtable", schema, fileFormat, 1);
    // access a single value in a struct
    for (int i = 0; i < records.size(); i++) {
        GenericRecord expectedStruct = (GenericRecord) records.get(i).getField("structofprimitives");
        List<Object[]> queryResult = shell.executeStatement(String.format("SELECT structofprimitives.key, structofprimitives.value FROM default.structtable LIMIT 1 OFFSET %d", i));
        Assert.assertEquals(expectedStruct.getField("key"), queryResult.get(0)[0]);
        Assert.assertEquals(expectedStruct.getField("value"), queryResult.get(0)[1]);
    }
}

Example 3 with GenericRecord

use of org.apache.iceberg.data.GenericRecord in project hive by apache.

the class TestHiveIcebergComplexTypeWrites method buildComplexTypeInnerQuery.

private StringBuilder buildComplexTypeInnerQuery(Object field, Type type) {
    StringBuilder query = new StringBuilder();
    if (type instanceof Types.ListType) {
        query.append("array(");
        List<Object> elements = (List<Object>) field;
        Assert.assertFalse("Hive can not handle empty array() inserts", elements.isEmpty());
        Type innerType = ((Types.ListType) type).fields().get(0).type();
        if (!elements.isEmpty()) {
            elements.forEach(e -> query.append(buildComplexTypeInnerQuery(e, innerType)));
            query.setLength(query.length() - 1);
        }
        query.append("),");
    } else if (type instanceof Types.MapType) {
        query.append("map(");
        Map<Object, Object> entries = (Map<Object, Object>) field;
        Type keyType = ((Types.MapType) type).fields().get(0).type();
        Type valueType = ((Types.MapType) type).fields().get(1).type();
        if (!entries.isEmpty()) {
            entries.entrySet().forEach(e -> query.append(buildComplexTypeInnerQuery(e.getKey(), keyType).append(buildComplexTypeInnerQuery(e.getValue(), valueType))));
            query.setLength(query.length() - 1);
        }
        query.append("),");
    } else if (type instanceof Types.StructType) {
        query.append("named_struct(");
        ((GenericRecord) field).struct().fields().stream().forEach(f -> query.append(buildComplexTypeInnerQuery(f.name(), Types.StringType.get())).append(buildComplexTypeInnerQuery(((GenericRecord) field).getField(f.name()), f.type())));
        query.setLength(query.length() - 1);
        query.append("),");
    } else if (type instanceof Types.StringType) {
        if (field != null) {
            query.append("'").append(field).append("',");
        }
    } else {
        throw new RuntimeException("Unsupported type in complex query build.");
    }
    return query;
}

Example 4 with GenericRecord

use of org.apache.iceberg.data.GenericRecord in project hive by apache.

the class TestHiveIcebergStorageHandlerLocalScan method testMapOfStructsInTable.

@Test
public void testMapOfStructsInTable() throws IOException {
    Schema schema = new Schema(required(1, "mapofstructs", Types.MapType.ofRequired(2, 3, Types.StringType.get(), Types.StructType.of(required(4, "something", Types.DoubleType.get()), required(5, "someone", Types.LongType.get()), required(6, "somewhere", Types.StringType.get())))));
    List<Record> records = testTables.createTableWithGeneratedRecords(shell, "maptable", schema, fileFormat, 1);
    // access a single element from a struct in a map
    for (int i = 0; i < records.size(); i++) {
        Map<?, ?> expectedMap = (Map<?, ?>) records.get(i).getField("mapofstructs");
        for (Map.Entry<?, ?> entry : expectedMap.entrySet()) {
            List<Object[]> queryResult = shell.executeStatement(String.format("SELECT mapofstructs[\"%s\"].something, " + "mapofstructs[\"%s\"].someone, mapofstructs[\"%s\"].somewhere FROM default.maptable LIMIT 1 " + "OFFSET %d", entry.getKey(), entry.getKey(), entry.getKey(), i));
            GenericRecord genericRecord = (GenericRecord) entry.getValue();
            Assert.assertEquals(genericRecord.getField("something"), queryResult.get(0)[0]);
            Assert.assertEquals(genericRecord.getField("someone"), queryResult.get(0)[1]);
            Assert.assertEquals(genericRecord.getField("somewhere"), queryResult.get(0)[2]);
        }
    }
}

Example 5 with GenericRecord

use of org.apache.iceberg.data.GenericRecord in project hive by apache.

the class TestHiveIcebergTypes method testStructOfArraysInTable.

@Test
public void testStructOfArraysInTable() throws IOException {
    Schema schema = new Schema(required(1, "structofarrays", Types.StructType.of(required(2, "names", Types.ListType.ofRequired(3, Types.StringType.get())), required(4, "birthdays", Types.ListType.ofRequired(5, Types.DateType.get())))));
    List<Record> records = testTables.createTableWithGeneratedRecords(shell, "structtable", schema, fileFormat, 1);
    // access an element of an array inside a struct
    for (int i = 0; i < records.size(); i++) {
        GenericRecord expectedStruct = (GenericRecord) records.get(i).getField("structofarrays");
        List<?> expectedList = (List<?>) expectedStruct.getField("names");
        for (int j = 0; j < expectedList.size(); j++) {
            List<Object[]> queryResult = shell.executeStatement(String.format("SELECT structofarrays.names[%d] FROM default.structtable LIMIT 1 OFFSET %d", j, i));
            Assert.assertEquals(expectedList.get(j), queryResult.get(0)[0]);
        }
        expectedList = (List<?>) expectedStruct.getField("birthdays");
        for (int j = 0; j < expectedList.size(); j++) {
            List<Object[]> queryResult = shell.executeStatement(String.format("SELECT structofarrays.birthdays[%d] FROM default.structtable LIMIT 1 OFFSET %d", j, i));
            Assert.assertEquals(expectedList.get(j).toString(), queryResult.get(0)[0]);
        }
    }
}