Examples with Partition io.trino.plugin.hive.metastore.Partition used on opensource projects

Example 11 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class GlueHiveMetastore method addPartitions.

@Override
public void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    try {
        stats.getCreatePartitions().call(() -> {
            List<Future<BatchCreatePartitionResult>> futures = new ArrayList<>();
            for (List<PartitionWithStatistics> partitionBatch : Lists.partition(partitions, BATCH_CREATE_PARTITION_MAX_PAGE_SIZE)) {
                List<PartitionInput> partitionInputs = mappedCopy(partitionBatch, partition -> GlueInputConverter.convertPartition(partition));
                long startTime = System.currentTimeMillis();
                futures.add(glueClient.batchCreatePartitionAsync(new BatchCreatePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionInputList(partitionInputs), new StatsRecordingAsyncHandler(stats.getBatchCreatePartition(), startTime)));
            }
            for (Future<BatchCreatePartitionResult> future : futures) {
                try {
                    BatchCreatePartitionResult result = future.get();
                    propagatePartitionErrorToTrinoException(databaseName, tableName, result.getErrors());
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new TrinoException(HIVE_METASTORE_ERROR, e);
                }
            }
            Set<GlueColumnStatisticsProvider.PartitionStatisticsUpdate> updates = partitions.stream().map(partitionWithStatistics -> new GlueColumnStatisticsProvider.PartitionStatisticsUpdate(partitionWithStatistics.getPartition(), partitionWithStatistics.getStatistics().getColumnStatistics())).collect(toImmutableSet());
            columnStatisticsProvider.updatePartitionStatistics(updates);
            return null;
        });
    } catch (AmazonServiceException | ExecutionException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 12 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class CachingHiveMetastore method loadPartitionsByNames.

private Map<HivePartitionName, Optional<Partition>> loadPartitionsByNames(Iterable<? extends HivePartitionName> partitionNames) {
    requireNonNull(partitionNames, "partitionNames is null");
    checkArgument(!Iterables.isEmpty(partitionNames), "partitionNames is empty");
    HivePartitionName firstPartition = Iterables.get(partitionNames, 0);
    HiveTableName hiveTableName = firstPartition.getHiveTableName();
    Optional<Table> table = getTable(hiveTableName.getDatabaseName(), hiveTableName.getTableName());
    if (table.isEmpty()) {
        return stream(partitionNames).collect(toImmutableMap(name -> name, name -> Optional.empty()));
    }
    List<String> partitionsToFetch = new ArrayList<>();
    for (HivePartitionName partitionName : partitionNames) {
        checkArgument(partitionName.getHiveTableName().equals(hiveTableName), "Expected table name %s but got %s", hiveTableName, partitionName.getHiveTableName());
        partitionsToFetch.add(partitionName.getPartitionName().orElseThrow());
    }
    ImmutableMap.Builder<HivePartitionName, Optional<Partition>> partitions = ImmutableMap.builder();
    Map<String, Optional<Partition>> partitionsByNames = delegate.getPartitionsByNames(table.get(), partitionsToFetch);
    for (HivePartitionName partitionName : partitionNames) {
        partitions.put(partitionName, partitionsByNames.getOrDefault(partitionName.getPartitionName().orElseThrow(), Optional.empty()));
    }
    return partitions.buildOrThrow();
}

Example 13 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class FileHiveMetastore method addPartitions.

@Override
public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    requireNonNull(databaseName, "databaseName is null");
    requireNonNull(tableName, "tableName is null");
    requireNonNull(partitions, "partitions is null");
    Table table = getRequiredTable(databaseName, tableName);
    TableType tableType = TableType.valueOf(table.getTableType());
    checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(tableType), "Invalid table type: %s", tableType);
    try {
        Map<Path, byte[]> schemaFiles = new LinkedHashMap<>();
        for (PartitionWithStatistics partitionWithStatistics : partitions) {
            Partition partition = partitionWithStatistics.getPartition();
            verifiedPartition(table, partition);
            Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
            Path schemaPath = getSchemaPath(PARTITION, partitionMetadataDirectory);
            if (metadataFileSystem.exists(schemaPath)) {
                throw new TrinoException(HIVE_METASTORE_ERROR, "Partition already exists");
            }
            byte[] schemaJson = partitionCodec.toJsonBytes(new PartitionMetadata(table, partitionWithStatistics));
            schemaFiles.put(schemaPath, schemaJson);
        }
        Set<Path> createdFiles = new LinkedHashSet<>();
        try {
            for (Entry<Path, byte[]> entry : schemaFiles.entrySet()) {
                try (OutputStream outputStream = metadataFileSystem.create(entry.getKey())) {
                    createdFiles.add(entry.getKey());
                    outputStream.write(entry.getValue());
                } catch (IOException e) {
                    throw new TrinoException(HIVE_METASTORE_ERROR, "Could not write partition schema", e);
                }
            }
        } catch (Throwable e) {
            for (Path createdFile : createdFiles) {
                try {
                    metadataFileSystem.delete(createdFile, false);
                } catch (IOException ignored) {
                }
            }
            throw e;
        }
    } catch (IOException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 14 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class BridgingHiveMetastore method getPartitionsByNames.

@Override
public Map<String, Optional<Partition>> getPartitionsByNames(Table table, List<String> partitionNames) {
    requireNonNull(partitionNames, "partitionNames is null");
    if (partitionNames.isEmpty()) {
        return ImmutableMap.of();
    }
    Map<String, List<String>> partitionNameToPartitionValuesMap = partitionNames.stream().collect(Collectors.toMap(identity(), HiveUtil::toPartitionValues));
    Map<List<String>, Partition> partitionValuesToPartitionMap = delegate.getPartitionsByNames(identity, table.getDatabaseName(), table.getTableName(), partitionNames).stream().map(partition -> fromMetastoreApiPartition(table, partition)).collect(Collectors.toMap(Partition::getValues, identity()));
    ImmutableMap.Builder<String, Optional<Partition>> resultBuilder = ImmutableMap.builder();
    for (Map.Entry<String, List<String>> entry : partitionNameToPartitionValuesMap.entrySet()) {
        Partition partition = partitionValuesToPartitionMap.get(entry.getValue());
        resultBuilder.put(entry.getKey(), Optional.ofNullable(partition));
    }
    return resultBuilder.buildOrThrow();
}

Example 15 with Partition

use of io.trino.plugin.hive.metastore.Partition in project trino by trinodb.

the class UnregisterPartitionProcedure method doUnregisterPartition.

private void doUnregisterPartition(ConnectorSession session, ConnectorAccessControl accessControl, String schemaName, String tableName, List<String> partitionColumn, List<String> partitionValues) {
    SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
    SemiTransactionalHiveMetastore metastore = hiveMetadataFactory.create(session.getIdentity(), true).getMetastore();
    Table table = metastore.getTable(schemaName, tableName).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    accessControl.checkCanDeleteFromTable(null, schemaTableName);
    checkIsPartitionedTable(table);
    checkPartitionColumns(table, partitionColumn);
    String partitionName = FileUtils.makePartName(partitionColumn, partitionValues);
    Partition partition = metastore.unsafeGetRawHiveMetastoreClosure().getPartition(schemaName, tableName, partitionValues).orElseThrow(() -> new TrinoException(NOT_FOUND, format("Partition '%s' does not exist", partitionName)));
    metastore.dropPartition(session, table.getDatabaseName(), table.getTableName(), partition.getValues(), false);
    metastore.commit();
}