Examples with Partition com.facebook.presto.hive.metastore.Partition used on opensource projects

Example 11 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class GlueHiveMetastore method batchGetPartition.

private List<Partition> batchGetPartition(String databaseName, String tableName, List<String> partitionNames) {
    try {
        List<Future<BatchGetPartitionResult>> batchGetPartitionFutures = new ArrayList<>();
        for (List<String> partitionNamesBatch : Lists.partition(partitionNames, BATCH_GET_PARTITION_MAX_PAGE_SIZE)) {
            List<PartitionValueList> partitionValuesBatch = mappedCopy(partitionNamesBatch, partitionName -> new PartitionValueList().withValues(toPartitionValues(partitionName)));
            batchGetPartitionFutures.add(glueClient.batchGetPartitionAsync(new BatchGetPartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionsToGet(partitionValuesBatch), stats.getBatchGetPartitions().metricsAsyncHandler()));
        }
        GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName);
        ImmutableList.Builder<Partition> resultsBuilder = ImmutableList.builderWithExpectedSize(partitionNames.size());
        for (Future<BatchGetPartitionResult> future : batchGetPartitionFutures) {
            future.get().getPartitions().stream().map(converter).forEach(resultsBuilder::add);
        }
        return resultsBuilder.build();
    } catch (AmazonServiceException | InterruptedException | ExecutionException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 12 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class FileHiveMetastore method addPartitions.

@Override
public synchronized void addPartitions(MetastoreContext metastoreContext, String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    requireNonNull(databaseName, "databaseName is null");
    requireNonNull(tableName, "tableName is null");
    requireNonNull(partitions, "partitions is null");
    Table table = getRequiredTable(metastoreContext, databaseName, tableName);
    checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE, MATERIALIZED_VIEW).contains(table.getTableType()), "Invalid table type: %s", table.getTableType());
    try {
        Map<Path, byte[]> schemaFiles = new LinkedHashMap<>();
        for (PartitionWithStatistics partitionWithStatistics : partitions) {
            Partition partition = partitionWithStatistics.getPartition();
            verifiedPartition(table, partition);
            Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
            Path schemaPath = new Path(partitionMetadataDirectory, PRESTO_SCHEMA_FILE_NAME);
            if (metadataFileSystem.exists(schemaPath)) {
                throw new PrestoException(HIVE_METASTORE_ERROR, "Partition already exists");
            }
            byte[] schemaJson = partitionCodec.toJsonBytes(new PartitionMetadata(table, partitionWithStatistics));
            schemaFiles.put(schemaPath, schemaJson);
        }
        Set<Path> createdFiles = new LinkedHashSet<>();
        try {
            for (Entry<Path, byte[]> entry : schemaFiles.entrySet()) {
                try (OutputStream outputStream = metadataFileSystem.create(entry.getKey())) {
                    createdFiles.add(entry.getKey());
                    outputStream.write(entry.getValue());
                } catch (IOException e) {
                    throw new PrestoException(HIVE_METASTORE_ERROR, "Could not write partition schema", e);
                }
            }
        } catch (Throwable e) {
            for (Path createdFile : createdFiles) {
                try {
                    metadataFileSystem.delete(createdFile, false);
                } catch (IOException ignored) {
                }
            }
            throw e;
        }
    } catch (IOException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 13 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class FileHiveMetastore method alterPartition.

@Override
public synchronized void alterPartition(MetastoreContext metastoreContext, String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) {
    Table table = getRequiredTable(metastoreContext, databaseName, tableName);
    Partition partition = partitionWithStatistics.getPartition();
    verifiedPartition(table, partition);
    Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
    writeSchemaFile("partition", partitionMetadataDirectory, partitionCodec, new PartitionMetadata(table, partitionWithStatistics), true);
}

Example 14 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class ThriftMetastoreUtil method fromMetastoreApiPartition.

public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition, PartitionMutator partitionMutator, ColumnConverter columnConverter) {
    StorageDescriptor storageDescriptor = partition.getSd();
    if (storageDescriptor == null) {
        throw new PrestoException(HIVE_INVALID_METADATA, "Partition does not contain a storage descriptor: " + partition);
    }
    Partition.Builder partitionBuilder = Partition.builder().setDatabaseName(partition.getDbName()).setTableName(partition.getTableName()).setValues(partition.getValues()).setColumns(storageDescriptor.getCols().stream().map(fieldSchema -> columnConverter.toColumn(fieldSchema)).collect(toList())).setParameters(partition.getParameters()).setCreateTime(partition.getCreateTime());
    // mutate apache partition to Presto partition
    partitionMutator.mutate(partitionBuilder, partition);
    fromMetastoreApiStorageDescriptor(storageDescriptor, partitionBuilder.getStorageBuilder(), format("%s.%s", partition.getTableName(), partition.getValues()));
    return partitionBuilder.build();
}

Example 15 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class HiveMetadata method finishInsert.

@Override
public Optional<ConnectorOutputMetadata> finishInsert(ConnectorSession session, ConnectorInsertTableHandle insertHandle, Collection<Slice> fragments) {
    HiveInsertTableHandle handle = (HiveInsertTableHandle) insertHandle;
    List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(partitionUpdateCodec::fromJson).collect(toList());
    HiveStorageFormat tableStorageFormat = handle.getTableStorageFormat();
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    Optional<Table> table = metastore.getTable(handle.getSchemaName(), handle.getTableName());
    if (!table.isPresent()) {
        throw new TableNotFoundException(new SchemaTableName(handle.getSchemaName(), handle.getTableName()));
    }
    if (!table.get().getStorage().getStorageFormat().getInputFormat().equals(tableStorageFormat.getInputFormat()) && respectTableFormat) {
        throw new PrestoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Table format changed during insert");
    }
    if (handle.getBucketProperty().isPresent()) {
        ImmutableList<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(handle, table.get(), partitionUpdates);
        // replace partitionUpdates before creating the empty files so that those files will be cleaned up if we end up rollback
        partitionUpdates = PartitionUpdate.mergePartitionUpdates(Iterables.concat(partitionUpdates, partitionUpdatesForMissingBuckets));
        for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
            Optional<Partition> partition = table.get().getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(buildPartitionObject(session.getQueryId(), table.get(), partitionUpdate));
            createEmptyFile(partitionUpdate.getWritePath(), table.get(), partition, partitionUpdate.getFileNames());
        }
    }
    for (PartitionUpdate partitionUpdate : partitionUpdates) {
        if (partitionUpdate.getName().isEmpty()) {
            // insert into unpartitioned table
            metastore.finishInsertIntoExistingTable(session, handle.getSchemaName(), handle.getTableName(), partitionUpdate.getWritePath(), partitionUpdate.getFileNames());
        } else if (!partitionUpdate.isNew()) {
            // insert into existing partition
            metastore.finishInsertIntoExistingPartition(session, handle.getSchemaName(), handle.getTableName(), toPartitionValues(partitionUpdate.getName()), partitionUpdate.getWritePath(), partitionUpdate.getFileNames());
        } else {
            // insert into new partition
            Partition partition = buildPartitionObject(session.getQueryId(), table.get(), partitionUpdate);
            if (!partition.getStorage().getStorageFormat().getInputFormat().equals(handle.getPartitionStorageFormat().getInputFormat()) && respectTableFormat) {
                throw new PrestoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Partition format changed during insert");
            }
            metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), partition, partitionUpdate.getWritePath());
        }
    }
    return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(Collectors.toList())));
}