Examples with Partition com.facebook.presto.hive.metastore.Partition used on opensource projects

Example 6 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class HiveMetadata method finishInsert.

@Override
public Optional<ConnectorOutputMetadata> finishInsert(ConnectorSession session, ConnectorInsertTableHandle insertHandle, Collection<Slice> fragments) {
    HiveInsertTableHandle handle = (HiveInsertTableHandle) insertHandle;
    List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(partitionUpdateCodec::fromJson).collect(toList());
    HiveStorageFormat tableStorageFormat = handle.getTableStorageFormat();
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    Optional<Table> table = metastore.getTable(handle.getSchemaName(), handle.getTableName());
    if (!table.isPresent()) {
        throw new TableNotFoundException(new SchemaTableName(handle.getSchemaName(), handle.getTableName()));
    }
    if (!table.get().getStorage().getStorageFormat().getInputFormat().equals(tableStorageFormat.getInputFormat()) && respectTableFormat) {
        throw new PrestoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Table format changed during insert");
    }
    if (handle.getBucketProperty().isPresent()) {
        ImmutableList<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(handle, table.get(), partitionUpdates);
        // replace partitionUpdates before creating the empty files so that those files will be cleaned up if we end up rollback
        partitionUpdates = PartitionUpdate.mergePartitionUpdates(Iterables.concat(partitionUpdates, partitionUpdatesForMissingBuckets));
        for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
            Optional<Partition> partition = table.get().getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(buildPartitionObject(session.getQueryId(), table.get(), partitionUpdate));
            createEmptyFile(partitionUpdate.getWritePath(), table.get(), partition, partitionUpdate.getFileNames());
        }
    }
    for (PartitionUpdate partitionUpdate : partitionUpdates) {
        if (partitionUpdate.getName().isEmpty()) {
            // insert into unpartitioned table
            metastore.finishInsertIntoExistingTable(session, handle.getSchemaName(), handle.getTableName(), partitionUpdate.getWritePath(), partitionUpdate.getFileNames());
        } else if (!partitionUpdate.isNew()) {
            // insert into existing partition
            metastore.finishInsertIntoExistingPartition(session, handle.getSchemaName(), handle.getTableName(), toPartitionValues(partitionUpdate.getName()), partitionUpdate.getWritePath(), partitionUpdate.getFileNames());
        } else {
            // insert into new partition
            Partition partition = buildPartitionObject(session.getQueryId(), table.get(), partitionUpdate);
            if (!partition.getStorage().getStorageFormat().getInputFormat().equals(handle.getPartitionStorageFormat().getInputFormat()) && respectTableFormat) {
                throw new PrestoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Partition format changed during insert");
            }
            metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), partition, partitionUpdate.getWritePath());
        }
    }
    return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(Collectors.toList())));
}

Example 7 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class HiveSplitManager method getPartitionMetadata.

private Iterable<HivePartitionMetadata> getPartitionMetadata(SemiTransactionalHiveMetastore metastore, Table table, SchemaTableName tableName, List<HivePartition> hivePartitions, Optional<HiveBucketProperty> bucketProperty) {
    if (hivePartitions.isEmpty()) {
        return ImmutableList.of();
    }
    if (hivePartitions.size() == 1) {
        HivePartition firstPartition = getOnlyElement(hivePartitions);
        if (firstPartition.getPartitionId().equals(UNPARTITIONED_ID)) {
            return ImmutableList.of(new HivePartitionMetadata(firstPartition, Optional.empty(), ImmutableMap.of()));
        }
    }
    Iterable<List<HivePartition>> partitionNameBatches = partitionExponentially(hivePartitions, minPartitionBatchSize, maxPartitionBatchSize);
    Iterable<List<HivePartitionMetadata>> partitionBatches = transform(partitionNameBatches, partitionBatch -> {
        Map<String, Optional<Partition>> batch = metastore.getPartitionsByNames(tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId));
        ImmutableMap.Builder<String, Partition> partitionBuilder = ImmutableMap.builder();
        for (Map.Entry<String, Optional<Partition>> entry : batch.entrySet()) {
            if (!entry.getValue().isPresent()) {
                throw new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available");
            }
            partitionBuilder.put(entry.getKey(), entry.getValue().get());
        }
        Map<String, Partition> partitions = partitionBuilder.build();
        Verify.verify(partitions.size() == partitionBatch.size());
        if (partitionBatch.size() != partitions.size()) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Expected %s partitions but found %s", partitionBatch.size(), partitions.size()));
        }
        ImmutableList.Builder<HivePartitionMetadata> results = ImmutableList.builder();
        for (HivePartition hivePartition : partitionBatch) {
            Partition partition = partitions.get(hivePartition.getPartitionId());
            if (partition == null) {
                throw new PrestoException(GENERIC_INTERNAL_ERROR, "Partition not loaded: " + hivePartition);
            }
            String protectMode = partition.getParameters().get(ProtectMode.PARAMETER_NAME);
            String partName = makePartName(table.getPartitionColumns(), partition.getValues());
            if (protectMode != null && getProtectModeFromString(protectMode).offline) {
                throw new PartitionOfflineException(tableName, partName, false, null);
            }
            String prestoOffline = partition.getParameters().get(PRESTO_OFFLINE);
            if (!isNullOrEmpty(prestoOffline)) {
                throw new PartitionOfflineException(tableName, partName, true, prestoOffline);
            }
            List<Column> tableColumns = table.getDataColumns();
            List<Column> partitionColumns = partition.getColumns();
            if ((tableColumns == null) || (partitionColumns == null)) {
                throw new PrestoException(HIVE_INVALID_METADATA, format("Table '%s' or partition '%s' has null columns", tableName, partName));
            }
            ImmutableMap.Builder<Integer, HiveType> columnCoercions = ImmutableMap.builder();
            for (int i = 0; i < min(partitionColumns.size(), tableColumns.size()); i++) {
                HiveType tableType = tableColumns.get(i).getType();
                HiveType partitionType = partitionColumns.get(i).getType();
                if (!tableType.equals(partitionType)) {
                    if (!coercionPolicy.canCoerce(partitionType, tableType)) {
                        throw new PrestoException(HIVE_PARTITION_SCHEMA_MISMATCH, format("" + "There is a mismatch between the table and partition schemas. " + "The types are incompatible and cannot be coerced. " + "The column '%s' in table '%s' is declared as type '%s', " + "but partition '%s' declared column '%s' as type '%s'.", tableColumns.get(i).getName(), tableName, tableType, partName, partitionColumns.get(i).getName(), partitionType));
                    }
                    columnCoercions.put(i, partitionType);
                }
            }
            Optional<HiveBucketProperty> partitionBucketProperty = partition.getStorage().getBucketProperty();
            checkCondition(partitionBucketProperty.equals(bucketProperty), HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH, "Hive table (%s) bucketing property (%s) does not match partition (%s) bucketing property (%s)", hivePartition.getTableName(), bucketProperty, hivePartition.getPartitionId(), partitionBucketProperty);
            results.add(new HivePartitionMetadata(hivePartition, Optional.of(partition), columnCoercions.build()));
        }
        return results.build();
    });
    return concat(partitionBatches);
}

Example 8 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class AbstractTestHiveClient method listAllDataPaths.

public static List<String> listAllDataPaths(SemiTransactionalHiveMetastore metastore, String schemaName, String tableName) {
    ImmutableList.Builder<String> locations = ImmutableList.builder();
    Table table = metastore.getTable(schemaName, tableName).get();
    if (table.getStorage().getLocation() != null) {
        // For partitioned table, there should be nothing directly under this directory.
        // But including this location in the set makes the directory content assert more
        // extensive, which is desirable.
        locations.add(table.getStorage().getLocation());
    }
    Optional<List<String>> partitionNames = metastore.getPartitionNames(schemaName, tableName);
    if (partitionNames.isPresent()) {
        metastore.getPartitionsByNames(schemaName, tableName, partitionNames.get()).values().stream().map(Optional::get).map(partition -> partition.getStorage().getLocation()).filter(location -> !location.startsWith(table.getStorage().getLocation())).forEach(locations::add);
    }
    return locations.build();
}

Example 9 with Partition

use of com.facebook.presto.hive.metastore.Partition in project presto by prestodb.

the class FileHiveMetastore method addPartitions.

@Override
public synchronized void addPartitions(String databaseName, String tableName, List<Partition> partitions) {
    requireNonNull(databaseName, "databaseName is null");
    requireNonNull(tableName, "tableName is null");
    requireNonNull(partitions, "partitions is null");
    Table table = getRequiredTable(databaseName, tableName);
    TableType tableType = TableType.valueOf(table.getTableType());
    checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(tableType), "Invalid table type: %s", tableType);
    try {
        Map<Path, byte[]> schemaFiles = new LinkedHashMap<>();
        for (Partition partition : partitions) {
            verifiedPartition(table, partition);
            Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
            Path schemaPath = new Path(partitionMetadataDirectory, PRESTO_SCHEMA_FILE_NAME);
            if (metadataFileSystem.exists(schemaPath)) {
                throw new PrestoException(HIVE_METASTORE_ERROR, "Partition already exists");
            }
            byte[] schemaJson = partitionCodec.toJsonBytes(new PartitionMetadata(table, partition));
            schemaFiles.put(schemaPath, schemaJson);
        }
        Set<Path> createdFiles = new LinkedHashSet<>();
        try {
            for (Entry<Path, byte[]> entry : schemaFiles.entrySet()) {
                try (OutputStream outputStream = metadataFileSystem.create(entry.getKey())) {
                    createdFiles.add(entry.getKey());
                    outputStream.write(entry.getValue());
                } catch (IOException e) {
                    throw new PrestoException(HIVE_METASTORE_ERROR, "Could not write partition schema", e);
                }
            }
        } catch (Throwable e) {
            for (Path createdFile : createdFiles) {
                try {
                    metadataFileSystem.delete(createdFile, false);
                } catch (IOException ignored) {
                }
            }
            throw e;
        }
    } catch (IOException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}