Examples with Partition com.facebook.presto.hive.metastore.Partition used on opensource projects

Example 21 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class HiveMetadata method finishCreateTable.

@Override
public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics) {
    HiveOutputTableHandle handle = (HiveOutputTableHandle) tableHandle;
    List<PartitionUpdate> partitionUpdates = getPartitionUpdates(session, fragments);
    Map<String, String> tableEncryptionParameters = ImmutableMap.of();
    Map<String, String> partitionEncryptionParameters = ImmutableMap.of();
    if (handle.getEncryptionInformation().isPresent()) {
        EncryptionInformation encryptionInformation = handle.getEncryptionInformation().get();
        if (encryptionInformation.getDwrfEncryptionMetadata().isPresent()) {
            if (handle.getPartitionedBy().isEmpty()) {
                tableEncryptionParameters = ImmutableMap.copyOf(encryptionInformation.getDwrfEncryptionMetadata().get().getExtraMetadata());
            } else {
                partitionEncryptionParameters = ImmutableMap.copyOf(encryptionInformation.getDwrfEncryptionMetadata().get().getExtraMetadata());
            }
        }
    }
    WriteInfo writeInfo = locationService.getQueryWriteInfo(handle.getLocationHandle());
    MetastoreContext metastoreContext = getMetastoreContext(session);
    Table table = buildTableObject(session.getQueryId(), handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), handle.getInputColumns(), handle.getTableStorageFormat(), handle.getPartitionedBy(), handle.getBucketProperty(), handle.getPreferredOrderingColumns(), ImmutableMap.<String, String>builder().putAll(handle.getAdditionalTableParameters()).putAll(tableEncryptionParameters).build(), writeInfo.getTargetPath(), MANAGED_TABLE, prestoVersion, metastoreContext);
    PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(handle.getTableOwner());
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    if (handle.getBucketProperty().isPresent() && isCreateEmptyBucketFiles(session)) {
        List<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(session, handle, table, partitionUpdates);
        // replace partitionUpdates before creating the zero-row files so that those files will be cleaned up if we end up rollback
        partitionUpdates = PartitionUpdate.mergePartitionUpdates(concat(partitionUpdates, partitionUpdatesForMissingBuckets));
        HdfsContext hdfsContext = new HdfsContext(session, table.getDatabaseName(), table.getTableName(), table.getStorage().getLocation(), true);
        for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
            Optional<Partition> partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(partitionObjectBuilder.buildPartitionObject(session, table, partitionUpdate, prestoVersion, partitionEncryptionParameters));
            zeroRowFileCreator.createFiles(session, hdfsContext, partitionUpdate.getWritePath(), getTargetFileNames(partitionUpdate.getFileWriteInfos()), getStorageFormat(partition, table), handle.getCompressionCodec(), getSchema(partition, table));
        }
    }
    Map<String, Type> columnTypes = handle.getInputColumns().stream().collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<List<String>, ComputedStatistics> partitionComputedStatistics = createComputedStatisticsToPartitionMap(computedStatistics, handle.getPartitionedBy(), columnTypes);
    PartitionStatistics tableStatistics;
    if (table.getPartitionColumns().isEmpty()) {
        HiveBasicStatistics basicStatistics = partitionUpdates.stream().map(PartitionUpdate::getStatistics).reduce((first, second) -> reduce(first, second, ADD)).orElse(createZeroStatistics());
        tableStatistics = createPartitionStatistics(session, basicStatistics, columnTypes, getColumnStatistics(partitionComputedStatistics, ImmutableList.of()));
    } else {
        tableStatistics = new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of());
    }
    metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics);
    if (handle.getPartitionedBy().isEmpty()) {
        return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID)));
    }
    if (isRespectTableFormat(session)) {
        verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat());
    }
    for (PartitionUpdate update : partitionUpdates) {
        Map<String, String> partitionParameters = partitionEncryptionParameters;
        if (isPreferManifestsToListFiles(session) && isFileRenamingEnabled(session)) {
            // Store list of file names and sizes in partition metadata when prefer_manifests_to_list_files and file_renaming_enabled are set to true
            partitionParameters = updatePartitionMetadataWithFileNamesAndSizes(update, partitionParameters);
        }
        Partition partition = partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion, partitionParameters);
        PartitionStatistics partitionStatistics = createPartitionStatistics(session, update.getStatistics(), columnTypes, getColumnStatistics(partitionComputedStatistics, partition.getValues()));
        metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), table.getStorage().getLocation(), true, partitionObjectBuilder.buildPartitionObject(session, table, update, prestoVersion, partitionParameters), update.getWritePath(), partitionStatistics);
    }
    return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(toList())));
}

Example 22 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class GlueHiveMetastore method getPartitions.

private List<Partition> getPartitions(String databaseName, String tableName, String expression) {
    if (partitionSegments == 1) {
        return getPartitions(databaseName, tableName, expression, null);
    }
    // Do parallel partition fetch.
    CompletionService<List<Partition>> completionService = new ExecutorCompletionService<>(executor);
    for (int i = 0; i < partitionSegments; i++) {
        Segment segment = new Segment().withSegmentNumber(i).withTotalSegments(partitionSegments);
        completionService.submit(() -> getPartitions(databaseName, tableName, expression, segment));
    }
    List<Partition> partitions = new ArrayList<>();
    try {
        for (int i = 0; i < partitionSegments; i++) {
            Future<List<Partition>> futurePartitions = completionService.take();
            partitions.addAll(futurePartitions.get());
        }
    } catch (ExecutionException | InterruptedException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new PrestoException(HIVE_METASTORE_ERROR, "Failed to fetch partitions from Glue Data Catalog", e);
    }
    partitions.sort(PARTITION_COMPARATOR);
    return partitions;
}

Example 23 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class GlueHiveMetastore method getPartitions.

private List<Partition> getPartitions(String databaseName, String tableName, String expression, @Nullable Segment segment) {
    try {
        GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName);
        ArrayList<Partition> partitions = new ArrayList<>();
        GetPartitionsRequest request = new GetPartitionsRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withExpression(expression).withSegment(segment).withMaxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS);
        do {
            GetPartitionsResult result = stats.getGetPartitions().record(() -> glueClient.getPartitions(request));
            request.setNextToken(result.getNextToken());
            partitions.ensureCapacity(partitions.size() + result.getPartitions().size());
            result.getPartitions().stream().map(converter).forEach(partitions::add);
        } while (request.getNextToken() != null);
        return partitions;
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 24 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class GlueHiveMetastore method dropPartition.

@Override
public void dropPartition(MetastoreContext metastoreContext, String databaseName, String tableName, List<String> parts, boolean deleteData) {
    Table table = getTableOrElseThrow(metastoreContext, databaseName, tableName);
    Partition partition = getPartition(metastoreContext, databaseName, tableName, parts).orElseThrow(() -> new PartitionNotFoundException(new SchemaTableName(databaseName, tableName), parts));
    try {
        stats.getDeletePartition().record(() -> glueClient.deletePartition(new DeletePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionValues(parts)));
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
    String partLocation = partition.getStorage().getLocation();
    if (deleteData && isManagedTable(table) && !isNullOrEmpty(partLocation)) {
        deleteDir(hdfsContext, hdfsEnvironment, new Path(partLocation), true);
    }
}

Example 25 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class GlueHiveMetastore method getPartitionNames.

@Override
public Optional<List<String>> getPartitionNames(MetastoreContext metastoreContext, String databaseName, String tableName) {
    Table table = getTableOrElseThrow(metastoreContext, databaseName, tableName);
    List<Partition> partitions = getPartitions(databaseName, tableName, WILDCARD_EXPRESSION);
    return Optional.of(buildPartitionNames(table.getPartitionColumns(), partitions));
}