Examples with Type org.apache.iceberg.types.Type used on opensource projects

Example 6 with Type

use of org.apache.iceberg.types.Type in project hive by apache.

the class TestHiveIcebergSelects method testSelectDistinctFromTable.

@Test
public void testSelectDistinctFromTable() throws IOException {
    for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
        Type type = SUPPORTED_TYPES.get(i);
        if ((type == Types.TimestampType.withZone() || type == Types.TimeType.get()) && isVectorized && fileFormat == FileFormat.ORC) {
            // ORC/TIMESTAMP_INSTANT and time are not supported vectorized types for Hive
            continue;
        }
        // TODO: remove this filter when issue #1881 is resolved
        if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
            continue;
        }
        String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
        String columnName = type.typeId().toString().toLowerCase() + "_column";
        Schema schema = new Schema(required(1, columnName, type));
        List<Record> records = TestHelper.generateRandomRecords(schema, 4, 0L);
        int size = records.stream().map(r -> r.getField(columnName)).collect(Collectors.toSet()).size();
        testTables.createTable(shell, tableName, schema, fileFormat, records);
        List<Object[]> queryResult = shell.executeStatement("select count(distinct(" + columnName + ")) from default." + tableName);
        int distinctIds = ((Long) queryResult.get(0)[0]).intValue();
        Assert.assertEquals(tableName, size, distinctIds);
    }
}

Example 7 with Type

use of org.apache.iceberg.types.Type in project hive by apache.

the class TestHiveIcebergSelects method testJoinTablesSupportedTypes.

@Test
public void testJoinTablesSupportedTypes() throws IOException {
    for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
        Type type = SUPPORTED_TYPES.get(i);
        if ((type == Types.TimestampType.withZone() || type == Types.TimeType.get()) && isVectorized && fileFormat == FileFormat.ORC) {
            // ORC/TIMESTAMP_INSTANT and time are not supported vectorized types for Hive
            continue;
        }
        // TODO: remove this filter when issue #1881 is resolved
        if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
            continue;
        }
        String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
        String columnName = type.typeId().toString().toLowerCase() + "_column";
        Schema schema = new Schema(required(1, columnName, type));
        List<Record> records = TestHelper.generateRandomRecords(schema, 1, 0L);
        testTables.createTable(shell, tableName, schema, fileFormat, records);
        List<Object[]> queryResult = shell.executeStatement("select s." + columnName + ", h." + columnName + " from default." + tableName + " s join default." + tableName + " h on h." + columnName + "=s." + columnName);
        Assert.assertEquals("Non matching record count for table " + tableName + " with type " + type, 1, queryResult.size());
    }
}

Example 8 with Type

use of org.apache.iceberg.types.Type in project hive by apache.

the class IcebergInternalRecordWrapper method converter.

private static Function<Object, Object> converter(Type type) {
    switch(type.typeId()) {
        case TIMESTAMP:
            return timestamp -> DateTimeUtil.timestamptzFromMicros((Long) timestamp);
        case STRUCT:
            IcebergInternalRecordWrapper wrapper = new IcebergInternalRecordWrapper(type.asStructType(), type.asStructType());
            return struct -> wrapper.wrap((StructLike) struct);
        case LIST:
            if (Type.TypeID.STRUCT.equals(type.asListType().elementType().typeId())) {
                StructType listElementSchema = type.asListType().elementType().asStructType();
                Function<Type, IcebergInternalRecordWrapper> createWrapper = t -> new IcebergInternalRecordWrapper(listElementSchema, listElementSchema);
                return list -> {
                    return ((List<?>) list).stream().map(item -> createWrapper.apply(type).wrap((StructLike) item)).collect(Collectors.toList());
                };
            }
            break;
        default:
    }
    return null;
}

Example 9 with Type

use of org.apache.iceberg.types.Type in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testCreateTableWithNotSupportedTypesWithAutoConversion.

@Test
public void testCreateTableWithNotSupportedTypesWithAutoConversion() {
    TableIdentifier identifier = TableIdentifier.of("default", "not_supported_types");
    // Can not create INTERVAL types from normal create table, so leave them out from this test
    Map<String, Type> notSupportedTypes = ImmutableMap.of("TINYINT", Types.IntegerType.get(), "SMALLINT", Types.IntegerType.get(), "VARCHAR(1)", Types.StringType.get(), "CHAR(1)", Types.StringType.get());
    shell.setHiveSessionValue(InputFormatConfig.SCHEMA_AUTO_CONVERSION, "true");
    for (String notSupportedType : notSupportedTypes.keySet()) {
        shell.executeStatement("CREATE EXTERNAL TABLE not_supported_types (not_supported " + notSupportedType + ") " + "STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + testTables.propertiesForCreateTableSQL(ImmutableMap.of()));
        org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
        Assert.assertEquals(notSupportedTypes.get(notSupportedType), icebergTable.schema().columns().get(0).type());
        shell.executeStatement("DROP TABLE not_supported_types");
    }
}

Example 10 with Type

use of org.apache.iceberg.types.Type in project hive by apache.

the class TestHiveIcebergComplexTypeWrites method buildComplexTypeInnerQuery.

private StringBuilder buildComplexTypeInnerQuery(Object field, Type type) {
    StringBuilder query = new StringBuilder();
    if (type instanceof Types.ListType) {
        query.append("array(");
        List<Object> elements = (List<Object>) field;
        Assert.assertFalse("Hive can not handle empty array() inserts", elements.isEmpty());
        Type innerType = ((Types.ListType) type).fields().get(0).type();
        if (!elements.isEmpty()) {
            elements.forEach(e -> query.append(buildComplexTypeInnerQuery(e, innerType)));
            query.setLength(query.length() - 1);
        }
        query.append("),");
    } else if (type instanceof Types.MapType) {
        query.append("map(");
        Map<Object, Object> entries = (Map<Object, Object>) field;
        Type keyType = ((Types.MapType) type).fields().get(0).type();
        Type valueType = ((Types.MapType) type).fields().get(1).type();
        if (!entries.isEmpty()) {
            entries.entrySet().forEach(e -> query.append(buildComplexTypeInnerQuery(e.getKey(), keyType).append(buildComplexTypeInnerQuery(e.getValue(), valueType))));
            query.setLength(query.length() - 1);
        }
        query.append("),");
    } else if (type instanceof Types.StructType) {
        query.append("named_struct(");
        ((GenericRecord) field).struct().fields().stream().forEach(f -> query.append(buildComplexTypeInnerQuery(f.name(), Types.StringType.get())).append(buildComplexTypeInnerQuery(((GenericRecord) field).getField(f.name()), f.type())));
        query.setLength(query.length() - 1);
        query.append("),");
    } else if (type instanceof Types.StringType) {
        if (field != null) {
            query.append("'").append(field).append("',");
        }
    } else {
        throw new RuntimeException("Unsupported type in complex query build.");
    }
    return query;
}