Examples with HiveType io.trino.plugin.hive.HiveType used on opensource projects

Example 11 with HiveType

use of io.trino.plugin.hive.HiveType in project trino by trinodb.

the class ThriftHiveMetastore method updateTableStatistics.

@Override
public void updateTableStatistics(HiveIdentity identity, String databaseName, String tableName, AcidTransaction transaction, Function<PartitionStatistics, PartitionStatistics> update) {
    Table originalTable = getTable(identity, databaseName, tableName).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    PartitionStatistics currentStatistics = getTableStatistics(identity, originalTable);
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    Table modifiedTable = originalTable.deepCopy();
    HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics();
    modifiedTable.setParameters(updateStatisticsParameters(modifiedTable.getParameters(), basicStatistics));
    if (transaction.isAcidTransactionRunning()) {
        modifiedTable.setWriteId(transaction.getWriteId());
    }
    alterTable(identity, databaseName, tableName, modifiedTable);
    io.trino.plugin.hive.metastore.Table table = fromMetastoreApiTable(modifiedTable);
    OptionalLong rowCount = basicStatistics.getRowCount();
    List<ColumnStatisticsObj> metastoreColumnStatistics = updatedStatistics.getColumnStatistics().entrySet().stream().flatMap(entry -> {
        Optional<Column> column = table.getColumn(entry.getKey());
        if (column.isEmpty() && isAvroTableWithSchemaSet(modifiedTable)) {
            // to store statistics for a column it does not know about.
            return Stream.of();
        }
        HiveType type = column.orElseThrow(() -> new IllegalStateException("Column not found: " + entry.getKey())).getType();
        return Stream.of(createMetastoreColumnStatistics(entry.getKey(), type, entry.getValue(), rowCount));
    }).collect(toImmutableList());
    if (!metastoreColumnStatistics.isEmpty()) {
        setTableColumnStatistics(identity, databaseName, tableName, metastoreColumnStatistics);
    }
    Set<String> removedColumnStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet());
    removedColumnStatistics.forEach(column -> deleteTableColumnStatistics(identity, databaseName, tableName, column));
}

Example 12 with HiveType

use of io.trino.plugin.hive.HiveType in project trino by trinodb.

the class ThriftHiveMetastore method updatePartitionStatistics.

@Override
public void updatePartitionStatistics(HiveIdentity identity, Table table, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) {
    List<Partition> partitions = getPartitionsByNames(identity, table.getDbName(), table.getTableName(), ImmutableList.of(partitionName));
    if (partitions.size() != 1) {
        throw new TrinoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName);
    }
    Partition originalPartition = getOnlyElement(partitions);
    PartitionStatistics currentStatistics = requireNonNull(getPartitionStatistics(identity, table, partitions).get(partitionName), "getPartitionStatistics() did not return statistics for partition");
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    Partition modifiedPartition = originalPartition.deepCopy();
    HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics();
    modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics));
    alterPartitionWithoutStatistics(identity, table.getDbName(), table.getTableName(), modifiedPartition);
    Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream().collect(toImmutableMap(FieldSchema::getName, schema -> HiveType.valueOf(schema.getType())));
    setPartitionColumnStatistics(identity, table.getDbName(), table.getTableName(), partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount());
    Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet());
    removedStatistics.forEach(column -> deletePartitionColumnStatistics(identity, table.getDbName(), table.getTableName(), partitionName, column));
}

Example 13 with HiveType

use of io.trino.plugin.hive.HiveType in project trino by trinodb.

the class ThriftHiveMetastore method storePartitionColumnStatistics.

private void storePartitionColumnStatistics(HiveIdentity identity, String databaseName, String tableName, String partitionName, PartitionWithStatistics partitionWithStatistics) {
    PartitionStatistics statistics = partitionWithStatistics.getStatistics();
    Map<String, HiveColumnStatistics> columnStatistics = statistics.getColumnStatistics();
    if (columnStatistics.isEmpty()) {
        return;
    }
    Map<String, HiveType> columnTypes = partitionWithStatistics.getPartition().getColumns().stream().collect(toImmutableMap(Column::getName, Column::getType));
    setPartitionColumnStatistics(identity, databaseName, tableName, partitionName, columnTypes, columnStatistics, statistics.getBasicStatistics().getRowCount());
}

Example 14 with HiveType

use of io.trino.plugin.hive.HiveType in project trino by trinodb.

the class OrcTypeToHiveTypeTranslator method fromOrcTypeToHiveType.

public static HiveType fromOrcTypeToHiveType(OrcType orcType, ColumnMetadata<OrcType> columnMetadata) {
    switch(orcType.getOrcTypeKind()) {
        case BOOLEAN:
            return HIVE_BOOLEAN;
        case FLOAT:
            return HIVE_FLOAT;
        case DOUBLE:
            return HIVE_DOUBLE;
        case BYTE:
            return HIVE_BYTE;
        case DATE:
            return HIVE_DATE;
        case SHORT:
            return HIVE_SHORT;
        case INT:
            return HIVE_INT;
        case LONG:
            return HIVE_LONG;
        case DECIMAL:
            checkArgument(orcType.getPrecision().isPresent(), "orcType.getPrecision() is not present");
            checkArgument(orcType.getScale().isPresent(), "orcType.getScale() is not present");
            return toHiveType(new DecimalTypeInfo(orcType.getPrecision().get(), orcType.getScale().get()));
        case TIMESTAMP:
            return HIVE_TIMESTAMP;
        case BINARY:
            return HIVE_BINARY;
        case CHAR:
        case VARCHAR:
        case STRING:
            return HIVE_STRING;
        case LIST:
            {
                HiveType elementType = fromOrcTypeToHiveType(columnMetadata.get(orcType.getFieldTypeIndex(0)), columnMetadata);
                return toHiveType(getListTypeInfo(elementType.getTypeInfo()));
            }
        case MAP:
            {
                HiveType keyType = getHiveType(orcType, 0, columnMetadata);
                HiveType elementType = getHiveType(orcType, 1, columnMetadata);
                return toHiveType(getMapTypeInfo(keyType.getTypeInfo(), elementType.getTypeInfo()));
            }
        case STRUCT:
            {
                ImmutableList.Builder<TypeInfo> fieldTypeInfo = ImmutableList.builder();
                for (int fieldId = 0; fieldId < orcType.getFieldCount(); fieldId++) {
                    fieldTypeInfo.add(getHiveType(orcType, fieldId, columnMetadata).getTypeInfo());
                }
                return toHiveType(getStructTypeInfo(orcType.getFieldNames(), fieldTypeInfo.build()));
            }
        case TIMESTAMP_INSTANT:
        case UNION:
            // unsupported
            break;
    }
    throw new VerifyException("Unhandled ORC type: " + orcType.getOrcTypeKind());
}

Example 15 with HiveType

use of io.trino.plugin.hive.HiveType in project trino by trinodb.

the class TestHiveBucketing method assertBucketsEqual.

private static void assertBucketsEqual(List<String> hiveTypeStrings, List<List<Object>> hiveValues, int bucketCount, Optional<Set<Integer>> expectedBucketsV1, Optional<Set<Integer>> expectedBucketsV2) {
    List<HiveType> hiveTypes = hiveTypeStrings.stream().map(HiveType::valueOf).collect(toImmutableList());
    List<TypeInfo> hiveTypeInfos = hiveTypes.stream().map(HiveType::getTypeInfo).collect(toImmutableList());
    List<Type> trinoTypes = hiveTypes.stream().map(type -> type.getType(TESTING_TYPE_MANAGER)).collect(toImmutableList());
    ImmutableList.Builder<List<NullableValue>> values = ImmutableList.builder();
    for (int i = 0; i < hiveValues.size(); i++) {
        List<Object> valueList = hiveValues.get(i);
        Type trinoType = trinoTypes.get(i);
        values.add(valueList.stream().map(value -> new NullableValue(trinoType, toNativeContainerValue(trinoType, value))).collect(toImmutableList()));
    }
    assertEquals(getHiveBuckets(BUCKETING_V1, bucketCount, hiveTypeInfos, values.build()), expectedBucketsV1);
    assertEquals(getHiveBuckets(BUCKETING_V2, bucketCount, hiveTypeInfos, values.build()), expectedBucketsV2);
}