Search in sources :

Example 1 with PartitionInput

use of com.amazonaws.services.glue.model.PartitionInput in project presto by prestodb.

the class GlueHiveMetastore method updatePartitionStatistics.

@Override
public void updatePartitionStatistics(MetastoreContext metastoreContext, String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) {
    PartitionStatistics currentStatistics = getPartitionStatistics(metastoreContext, databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName);
    if (currentStatistics == null) {
        throw new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Statistics result does not contain entry for partition: " + partitionName);
    }
    PartitionStatistics updatedStatistics = update.apply(currentStatistics);
    if (!updatedStatistics.getColumnStatistics().isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics");
    }
    List<String> partitionValues = toPartitionValues(partitionName);
    Partition partition = getPartition(metastoreContext, databaseName, tableName, partitionValues).orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues));
    try {
        PartitionInput partitionInput = GlueInputConverter.convertPartition(partition);
        partitionInput.setParameters(updateStatisticsParameters(partition.getParameters(), updatedStatistics.getBasicStatistics()));
        stats.getUpdatePartition().record(() -> glueClient.updatePartition(new UpdatePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionValueList(partition.getValues()).withPartitionInput(partitionInput)));
    } catch (EntityNotFoundException e) {
        throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partitionValues);
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : Partition(com.facebook.presto.hive.metastore.Partition) PartitionNotFoundException(com.facebook.presto.hive.PartitionNotFoundException) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) AmazonServiceException(com.amazonaws.AmazonServiceException) PrestoException(com.facebook.presto.spi.PrestoException) UpdatePartitionRequest(com.amazonaws.services.glue.model.UpdatePartitionRequest) EntityNotFoundException(com.amazonaws.services.glue.model.EntityNotFoundException) SchemaTableName(com.facebook.presto.spi.SchemaTableName) PartitionInput(com.amazonaws.services.glue.model.PartitionInput)

Example 2 with PartitionInput

use of com.amazonaws.services.glue.model.PartitionInput in project presto by prestodb.

the class GlueInputConverter method convertPartition.

public static PartitionInput convertPartition(PartitionWithStatistics partitionWithStatistics) {
    PartitionInput input = convertPartition(partitionWithStatistics.getPartition());
    PartitionStatistics statistics = partitionWithStatistics.getStatistics();
    if (!statistics.getColumnStatistics().isEmpty()) {
        throw new PrestoException(NOT_SUPPORTED, "Glue metastore does not support column level statistics");
    }
    input.setParameters(updateStatisticsParameters(input.getParameters(), statistics.getBasicStatistics()));
    return input;
}
Also used : PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) PrestoException(com.facebook.presto.spi.PrestoException) PartitionInput(com.amazonaws.services.glue.model.PartitionInput)

Example 3 with PartitionInput

use of com.amazonaws.services.glue.model.PartitionInput in project presto by prestodb.

the class GlueHiveMetastore method alterPartition.

@Override
public void alterPartition(MetastoreContext metastoreContext, String databaseName, String tableName, PartitionWithStatistics partition) {
    try {
        PartitionInput newPartition = GlueInputConverter.convertPartition(partition);
        stats.getUpdatePartition().record(() -> glueClient.updatePartition(new UpdatePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionInput(newPartition).withPartitionValueList(partition.getPartition().getValues())));
    } catch (EntityNotFoundException e) {
        throw new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), partition.getPartition().getValues());
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : PartitionNotFoundException(com.facebook.presto.hive.PartitionNotFoundException) AmazonServiceException(com.amazonaws.AmazonServiceException) PrestoException(com.facebook.presto.spi.PrestoException) UpdatePartitionRequest(com.amazonaws.services.glue.model.UpdatePartitionRequest) EntityNotFoundException(com.amazonaws.services.glue.model.EntityNotFoundException) SchemaTableName(com.facebook.presto.spi.SchemaTableName) PartitionInput(com.amazonaws.services.glue.model.PartitionInput)

Example 4 with PartitionInput

use of com.amazonaws.services.glue.model.PartitionInput in project presto by prestodb.

the class GlueHiveMetastore method addPartitions.

@Override
public void addPartitions(MetastoreContext metastoreContext, String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    try {
        List<Future<BatchCreatePartitionResult>> futures = new ArrayList<>();
        for (List<PartitionWithStatistics> partitionBatch : Lists.partition(partitions, BATCH_CREATE_PARTITION_MAX_PAGE_SIZE)) {
            List<PartitionInput> partitionInputs = mappedCopy(partitionBatch, GlueInputConverter::convertPartition);
            futures.add(glueClient.batchCreatePartitionAsync(new BatchCreatePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionInputList(partitionInputs), stats.getBatchCreatePartitions().metricsAsyncHandler()));
        }
        for (Future<BatchCreatePartitionResult> future : futures) {
            BatchCreatePartitionResult result = future.get();
            propagatePartitionErrorToPrestoException(databaseName, tableName, result.getErrors());
        }
    } catch (AmazonServiceException | InterruptedException | ExecutionException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}
Also used : ArrayList(java.util.ArrayList) PrestoException(com.facebook.presto.spi.PrestoException) PartitionInput(com.amazonaws.services.glue.model.PartitionInput) GlueInputConverter(com.facebook.presto.hive.metastore.glue.converter.GlueInputConverter) BatchCreatePartitionRequest(com.amazonaws.services.glue.model.BatchCreatePartitionRequest) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) BatchCreatePartitionResult(com.amazonaws.services.glue.model.BatchCreatePartitionResult) AmazonServiceException(com.amazonaws.AmazonServiceException) Future(java.util.concurrent.Future) ExecutionException(java.util.concurrent.ExecutionException)

Example 5 with PartitionInput

use of com.amazonaws.services.glue.model.PartitionInput in project presto by prestodb.

the class GlueInputConverter method convertPartition.

public static PartitionInput convertPartition(Partition partition) {
    PartitionInput input = new PartitionInput();
    input.setValues(partition.getValues());
    input.setStorageDescriptor(convertStorage(partition.getStorage(), partition.getColumns()));
    input.setParameters(partition.getParameters());
    return input;
}
Also used : PartitionInput(com.amazonaws.services.glue.model.PartitionInput)

Aggregations

PartitionInput (com.amazonaws.services.glue.model.PartitionInput)6 PrestoException (com.facebook.presto.spi.PrestoException)4 AmazonServiceException (com.amazonaws.AmazonServiceException)3 EntityNotFoundException (com.amazonaws.services.glue.model.EntityNotFoundException)2 UpdatePartitionRequest (com.amazonaws.services.glue.model.UpdatePartitionRequest)2 PartitionNotFoundException (com.facebook.presto.hive.PartitionNotFoundException)2 PartitionStatistics (com.facebook.presto.hive.metastore.PartitionStatistics)2 SchemaTableName (com.facebook.presto.spi.SchemaTableName)2 BatchCreatePartitionRequest (com.amazonaws.services.glue.model.BatchCreatePartitionRequest)1 BatchCreatePartitionResult (com.amazonaws.services.glue.model.BatchCreatePartitionResult)1 Partition (com.facebook.presto.hive.metastore.Partition)1 PartitionWithStatistics (com.facebook.presto.hive.metastore.PartitionWithStatistics)1 GlueInputConverter (com.facebook.presto.hive.metastore.glue.converter.GlueInputConverter)1 ArrayList (java.util.ArrayList)1 ExecutionException (java.util.concurrent.ExecutionException)1 Future (java.util.concurrent.Future)1 Test (org.testng.annotations.Test)1