Examples with HiveType com.facebook.presto.hive.HiveType used on opensource projects

Example 1 with HiveType

use of com.facebook.presto.hive.HiveType in project presto by prestodb.

the class AggregatedOrcPageSource method writeMinMax.

private void writeMinMax(int columnIndex, Type type, HiveType hiveType, BlockBuilder blockBuilder, boolean isMin) {
    ColumnStatistics columnStatistics = footer.getFileStats().get(columnIndex + 1);
    OrcType orcType = footer.getTypes().get(columnIndex + 1);
    if (type instanceof FixedWidthType) {
        completedBytes += ((FixedWidthType) type).getFixedSize();
    }
    String orcNoMinMaxMessage = "No min/max found for orc file. Set session property hive.pushdown_partial_aggregations_into_scan=false and execute query again";
    switch(orcType.getOrcTypeKind()) {
        case SHORT:
        case INT:
        case LONG:
            {
                Long value = isMin ? columnStatistics.getIntegerStatistics().getMin() : columnStatistics.getIntegerStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(value);
                }
                break;
            }
        case TIMESTAMP:
        case DATE:
            {
                Integer value = isMin ? columnStatistics.getDateStatistics().getMin() : columnStatistics.getDateStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(Long.valueOf(value));
                }
                break;
            }
        case VARCHAR:
        case CHAR:
        case STRING:
            {
                Slice value = isMin ? columnStatistics.getStringStatistics().getMin() : columnStatistics.getStringStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeBytes(value, 0, value.length()).closeEntry();
                    completedBytes += value.length();
                }
                break;
            }
        case FLOAT:
            {
                Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    blockBuilder.writeLong(floatToRawIntBits(value.floatValue()));
                }
                break;
            }
        case DOUBLE:
            {
                Double value = isMin ? columnStatistics.getDoubleStatistics().getMin() : columnStatistics.getDoubleStatistics().getMax();
                if (value == null) {
                    throw new UnsupportedOperationException(orcNoMinMaxMessage);
                } else {
                    type.writeDouble(blockBuilder, value);
                }
                break;
            }
        case DECIMAL:
            BigDecimal value = isMin ? columnStatistics.getDecimalStatistics().getMin() : columnStatistics.getDecimalStatistics().getMax();
            if (value == null) {
                throw new UnsupportedOperationException(orcNoMinMaxMessage);
            } else {
                Type definedType = hiveType.getType(typeManager);
                if (Decimals.isShortDecimal(definedType)) {
                    blockBuilder.writeLong(value.unscaledValue().longValue());
                } else {
                    type.writeSlice(blockBuilder, Decimals.encodeUnscaledValue(value.unscaledValue()));
                }
            }
            break;
        case BYTE:
        case BOOLEAN:
        case BINARY:
        case UNION:
        case LIST:
        case STRUCT:
        case MAP:
        default:
            throw new IllegalArgumentException("Unsupported type: " + orcType.getOrcTypeKind());
    }
}

Example 2 with HiveType

use of com.facebook.presto.hive.HiveType in project presto by prestodb.

the class ThriftHiveMetastore method storePartitionColumnStatistics.

private void storePartitionColumnStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, String partitionName, PartitionWithStatistics partitionWithStatistics) {
    PartitionStatistics statistics = partitionWithStatistics.getStatistics();
    Map<String, HiveColumnStatistics> columnStatistics = statistics.getColumnStatistics();
    if (columnStatistics.isEmpty()) {
        return;
    }
    Map<String, HiveType> columnTypes = partitionWithStatistics.getPartition().getColumns().stream().collect(toImmutableMap(Column::getName, Column::getType));
    setPartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionName, columnTypes, columnStatistics, statistics.getBasicStatistics().getRowCount());
}

Example 3 with HiveType

use of com.facebook.presto.hive.HiveType in project presto by prestodb.

the class HiveParquetDereferencePushDown method createSubfieldColumnHandle.

@Override
protected ColumnHandle createSubfieldColumnHandle(ColumnHandle baseColumnHandle, Subfield subfield, Type subfieldDataType, String subfieldColumnName) {
    if (baseColumnHandle == null) {
        throw new IllegalArgumentException("nested column [" + subfield + "]'s base column " + subfield.getRootName() + " is not present in table scan output");
    }
    HiveColumnHandle hiveBaseColumnHandle = (HiveColumnHandle) baseColumnHandle;
    Optional<HiveType> nestedColumnHiveType = hiveBaseColumnHandle.getHiveType().findChildType(subfield.getPath().stream().map(p -> ((Subfield.NestedField) p).getName()).collect(Collectors.toList()));
    if (!nestedColumnHiveType.isPresent()) {
        throw new IllegalArgumentException("nested column [" + subfield + "] type is not present in Hive column type");
    }
    // Create column handle for subfield column
    return new HiveColumnHandle(subfieldColumnName, nestedColumnHiveType.get(), subfieldDataType.getTypeSignature(), -1, SYNTHESIZED, Optional.of("nested column pushdown"), ImmutableList.of(subfield), Optional.empty());
}

Example 4 with HiveType

use of com.facebook.presto.hive.HiveType in project presto by prestodb.

the class ThriftHiveMetastore method updatePartitionStatistics.

@Override
public synchronized void updatePartitionStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) {
    PartitionStatistics currentStatistics = requireNonNull(getPartitionStatistics(metastoreContext, databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName), "getPartitionStatistics() returned null");
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    List<Partition> partitions = getPartitionsByNames(metastoreContext, databaseName, tableName, ImmutableList.of(partitionName));
    if (partitions.size() != 1) {
        throw new PrestoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName);
    }
    Partition originalPartition = getOnlyElement(partitions);
    Partition modifiedPartition = originalPartition.deepCopy();
    HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics();
    modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics));
    alterPartitionWithoutStatistics(metastoreContext, databaseName, tableName, modifiedPartition);
    Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream().collect(toImmutableMap(FieldSchema::getName, schema -> metastoreContext.getColumnConverter().toColumn(schema).getType()));
    setPartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount());
    Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet());
    removedStatistics.forEach(column -> deletePartitionColumnStatistics(metastoreContext, databaseName, tableName, partitionName, column));
}

Example 5 with HiveType

use of com.facebook.presto.hive.HiveType in project presto by prestodb.

the class MetastoreUtil method getHiveSchema.

public static Properties getHiveSchema(Storage storage, List<Column> partitionDataColumns, List<Column> tableDataColumns, Map<String, String> tableParameters, String databaseName, String tableName, List<String> partitionKeyNames, List<HiveType> partitionKeyTypes) {
    // Mimics function in Hive:
    // MetaStoreUtils.getSchema(StorageDescriptor, StorageDescriptor, Map<String, String>, String, String, List<FieldSchema>)
    Properties schema = new Properties();
    schema.setProperty(FILE_INPUT_FORMAT, storage.getStorageFormat().getInputFormat());
    schema.setProperty(FILE_OUTPUT_FORMAT, storage.getStorageFormat().getOutputFormat());
    schema.setProperty(META_TABLE_NAME, databaseName + "." + tableName);
    schema.setProperty(META_TABLE_LOCATION, storage.getLocation());
    if (storage.getBucketProperty().isPresent()) {
        List<String> bucketedBy = storage.getBucketProperty().get().getBucketedBy();
        if (!bucketedBy.isEmpty()) {
            schema.setProperty(BUCKET_FIELD_NAME, Joiner.on(",").join(bucketedBy));
        }
        schema.setProperty(BUCKET_COUNT, Integer.toString(storage.getBucketProperty().get().getBucketCount()));
    } else {
        schema.setProperty(BUCKET_COUNT, "0");
    }
    for (Entry<String, String> param : storage.getSerdeParameters().entrySet()) {
        schema.setProperty(param.getKey(), (param.getValue() != null) ? param.getValue() : "");
    }
    schema.setProperty(SERIALIZATION_LIB, storage.getStorageFormat().getSerDe());
    StringBuilder columnNameBuilder = new StringBuilder();
    StringBuilder columnTypeBuilder = new StringBuilder();
    StringBuilder columnCommentBuilder = new StringBuilder();
    boolean first = true;
    for (Column column : tableDataColumns) {
        if (!first) {
            columnNameBuilder.append(",");
            columnTypeBuilder.append(":");
            columnCommentBuilder.append('\0');
        }
        columnNameBuilder.append(column.getName());
        columnTypeBuilder.append(column.getType());
        columnCommentBuilder.append(column.getComment().orElse(""));
        first = false;
    }
    String columnNames = columnNameBuilder.toString();
    String columnTypes = columnTypeBuilder.toString();
    schema.setProperty(META_TABLE_COLUMNS, columnNames);
    schema.setProperty(META_TABLE_COLUMN_TYPES, columnTypes);
    schema.setProperty("columns.comments", columnCommentBuilder.toString());
    schema.setProperty(SERIALIZATION_DDL, toThriftDdl(tableName, partitionDataColumns));
    String partString = "";
    String partStringSep = "";
    String partTypesString = "";
    String partTypesStringSep = "";
    for (int index = 0; index < partitionKeyNames.size(); ++index) {
        String name = partitionKeyNames.get(index);
        HiveType type = partitionKeyTypes.get(index);
        partString += partStringSep;
        partString += name;
        partTypesString += partTypesStringSep;
        partTypesString += type.getHiveTypeName().toString();
        if (partStringSep.length() == 0) {
            partStringSep = "/";
            partTypesStringSep = ":";
        }
    }
    if (partString.length() > 0) {
        schema.setProperty(META_TABLE_PARTITION_COLUMNS, partString);
        schema.setProperty(META_TABLE_PARTITION_COLUMN_TYPES, partTypesString);
    }
    if (tableParameters != null) {
        for (Entry<String, String> entry : tableParameters.entrySet()) {
            // add non-null parameters to the schema
            if (entry.getValue() != null) {
                schema.setProperty(entry.getKey(), entry.getValue());
            }
        }
    }
    return schema;
}