Examples with PartitionStatistics io.prestosql.plugin.hive.PartitionStatistics used on opensource projects

Example 11 with PartitionStatistics

use of io.prestosql.plugin.hive.PartitionStatistics in project hetu-core by openlookeng.

the class MetastoreHiveStatisticsProvider method getPartitionRowCount.

private static OptionalDouble getPartitionRowCount(String partitionName, Map<String, PartitionStatistics> statistics) {
    PartitionStatistics partitionStatistics = statistics.get(partitionName);
    if (partitionStatistics == null) {
        return OptionalDouble.empty();
    }
    OptionalLong rowCount = partitionStatistics.getBasicStatistics().getRowCount();
    if (rowCount.isPresent()) {
        verify(rowCount.getAsLong() >= 0, "rowCount must be greater than or equal to zero");
        return OptionalDouble.of(rowCount.getAsLong());
    }
    return OptionalDouble.empty();
}

Example 12 with PartitionStatistics

use of io.prestosql.plugin.hive.PartitionStatistics in project hetu-core by openlookeng.

the class SemiTransactionalHiveMetastore method finishInsertIntoExistingPartition.

public synchronized void finishInsertIntoExistingPartition(ConnectorSession session, String databaseName, String tableName, List<String> partitionValues, Path currentLocation, List<String> fileNames, PartitionStatistics statisticsUpdate, HiveACIDWriteType acidWriteType) {
    setShared();
    isVacuumIncluded |= HiveACIDWriteType.isVacuum(acidWriteType);
    HiveIdentity identity = new HiveIdentity(session);
    SchemaTableName schemaTableName = new SchemaTableName(databaseName, tableName);
    Map<List<String>, Action<PartitionAndMore>> partitionActionsOfTable = partitionActions.computeIfAbsent(schemaTableName, k -> new LinkedHashMap<>());
    Action<PartitionAndMore> oldPartitionAction = partitionActionsOfTable.get(partitionValues);
    if (oldPartitionAction == null) {
        Partition partition = delegate.getPartition(identity, databaseName, tableName, partitionValues).orElseThrow(() -> new PartitionNotFoundException(schemaTableName, partitionValues));
        String partitionName = getPartitionName(identity, databaseName, tableName, partitionValues);
        PartitionStatistics mergedStatistics = statisticsUpdate;
        boolean updateStats = canUpdateStats(session, acidWriteType);
        if (updateStats) {
            PartitionStatistics currentStatistics = closure.getPartitionStatistics(identity, databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName);
            if (currentStatistics == null) {
                throw new PrestoException(HiveErrorCode.HIVE_METASTORE_ERROR, "currentStatistics is null");
            }
            mergedStatistics = merge(currentStatistics, statisticsUpdate);
        }
        HdfsContext context = new HdfsContext(session, databaseName, tableName);
        partitionActionsOfTable.put(partitionValues, new Action<>(ActionType.INSERT_EXISTING, new PartitionAndMore(identity, partition, currentLocation, Optional.of(fileNames), mergedStatistics, statisticsUpdate, updateStats), context, identity));
        return;
    }
    switch(oldPartitionAction.getType()) {
        case DROP:
            throw new PartitionNotFoundException(schemaTableName, partitionValues);
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
            throw new UnsupportedOperationException("Inserting into a partition that were added, altered, or inserted into in the same transaction is not supported");
        default:
            throw new IllegalStateException("Unknown action type");
    }
}

Example 13 with PartitionStatistics

use of io.prestosql.plugin.hive.PartitionStatistics in project hetu-core by openlookeng.

the class CachingHiveMetastore method loadPartitionColumnStatistics.

private Map<WithIdentity<HivePartitionName>, WithValidation<Table, PartitionStatistics>> loadPartitionColumnStatistics(Iterable<? extends WithIdentity<HivePartitionName>> keys) {
    SetMultimap<WithIdentity<HiveTableName>, WithIdentity<HivePartitionName>> tablePartitions = stream(keys).collect(toImmutableSetMultimap(value -> new WithIdentity<>(value.getIdentity(), value.getKey().getHiveTableName()), key -> key));
    ImmutableMap.Builder<WithIdentity<HivePartitionName>, WithValidation<Table, PartitionStatistics>> result = ImmutableMap.builder();
    tablePartitions.keySet().forEach(tableName -> {
        Set<WithIdentity<HivePartitionName>> partitionNames = tablePartitions.get(tableName);
        Set<String> partitionNameStrings = partitionNames.stream().map(partitionName -> partitionName.getKey().getPartitionName().get()).collect(toImmutableSet());
        Table table = getExistingTable(tableName.getIdentity(), tableName.getKey().getDatabaseName(), tableName.getKey().getTableName());
        List<Partition> partitions = getExistingPartitionsByNames(tableName.getIdentity(), table, ImmutableList.copyOf(partitionNameStrings));
        Map<String, PartitionStatistics> statisticsByPartitionName = delegate.getPartitionStatistics(tableName.getIdentity(), table, partitions);
        for (WithIdentity<HivePartitionName> partitionName : partitionNames) {
            String stringNameForPartition = partitionName.getKey().getPartitionName().get();
            PartitionStatistics value = statisticsByPartitionName.get(stringNameForPartition);
            if (value == null) {
                throw new PrestoException(HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + stringNameForPartition);
            }
            result.put(partitionName, new WithValidation<>(getCacheValidationPartitionParams(table, value.getBasicStatistics()), value));
        }
    });
    return result.build();
}

Example 14 with PartitionStatistics

use of io.prestosql.plugin.hive.PartitionStatistics in project hetu-core by openlookeng.

the class CachingHiveMetastore method loadPartitionColumnStatistics.

private WithValidation<Table, PartitionStatistics> loadPartitionColumnStatistics(WithIdentity<HivePartitionName> partition) {
    HiveTableName hiveTableName = partition.getKey().getHiveTableName();
    HiveIdentity identity = partition.getIdentity();
    Table table = getExistingTable(identity, hiveTableName.getDatabaseName(), hiveTableName.getTableName());
    String partitionName = partition.getKey().getPartitionName().get();
    Map<String, PartitionStatistics> partitionStatistics = delegate.getPartitionStatistics(identity, table, ImmutableList.of(getExistingPartition(identity, table, partition.getKey().getPartitionValues())));
    if (!partitionStatistics.containsKey(partitionName)) {
        throw new PrestoException(HiveErrorCode.HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + partition.getKey().getPartitionName());
    }
    PartitionStatistics value = partitionStatistics.get(partitionName);
    return new WithValidation<>(getCacheValidationPartitionParams(table, value.getBasicStatistics()), value);
}

Example 15 with PartitionStatistics

use of io.prestosql.plugin.hive.PartitionStatistics in project hetu-core by openlookeng.

the class GlueHiveMetastore method updatePartitionStatistics.

@Override
public void updatePartitionStatistics(HiveIdentity identity, String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) {
    List<String> partitionValues = toPartitionValues(partitionName);
    Partition partition = getPartition(identity, databaseName, tableName, partitionValues).orElseThrow(() -> new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + partitionName));
    PartitionStatistics currentStatistics = getPartitionStatistics(partition);
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    if (!updatedStatistics.getColumnStatistics().isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics");
    }
    try {
        PartitionInput partitionInput = GlueInputConverter.convertPartition(partition);
        partitionInput.setParameters(ThriftMetastoreUtil.updateStatisticsParameters(partition.getParameters(), updatedStatistics.getBasicStatistics()));
        glueClient.updatePartition(new UpdatePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionValueList(partition.getValues()).withPartitionInput(partitionInput));
    } catch (EntityNotFoundException e) {
        throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues);
    } catch (AmazonServiceException e) {
        throw new PrestoException(HiveErrorCode.HIVE_METASTORE_ERROR, e);
    }
}