Examples with PartitionWithStatistics io.trino.plugin.hive.metastore.PartitionWithStatistics used on opensource projects

Example 1 with PartitionWithStatistics

use of io.trino.plugin.hive.metastore.PartitionWithStatistics in project trino by trinodb.

the class GlueHiveMetastore method addPartitions.

@Override
public void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    try {
        stats.getCreatePartitions().call(() -> {
            List<Future<BatchCreatePartitionResult>> futures = new ArrayList<>();
            for (List<PartitionWithStatistics> partitionBatch : Lists.partition(partitions, BATCH_CREATE_PARTITION_MAX_PAGE_SIZE)) {
                List<PartitionInput> partitionInputs = mappedCopy(partitionBatch, partition -> GlueInputConverter.convertPartition(partition));
                long startTime = System.currentTimeMillis();
                futures.add(glueClient.batchCreatePartitionAsync(new BatchCreatePartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionInputList(partitionInputs), new StatsRecordingAsyncHandler(stats.getBatchCreatePartition(), startTime)));
            }
            for (Future<BatchCreatePartitionResult> future : futures) {
                try {
                    BatchCreatePartitionResult result = future.get();
                    propagatePartitionErrorToTrinoException(databaseName, tableName, result.getErrors());
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new TrinoException(HIVE_METASTORE_ERROR, e);
                }
            }
            Set<GlueColumnStatisticsProvider.PartitionStatisticsUpdate> updates = partitions.stream().map(partitionWithStatistics -> new GlueColumnStatisticsProvider.PartitionStatisticsUpdate(partitionWithStatistics.getPartition(), partitionWithStatistics.getStatistics().getColumnStatistics())).collect(toImmutableSet());
            columnStatisticsProvider.updatePartitionStatistics(updates);
            return null;
        });
    } catch (AmazonServiceException | ExecutionException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 2 with PartitionWithStatistics

use of io.trino.plugin.hive.metastore.PartitionWithStatistics in project trino by trinodb.

the class FileHiveMetastore method addPartitions.

@Override
public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitions) {
    requireNonNull(databaseName, "databaseName is null");
    requireNonNull(tableName, "tableName is null");
    requireNonNull(partitions, "partitions is null");
    Table table = getRequiredTable(databaseName, tableName);
    TableType tableType = TableType.valueOf(table.getTableType());
    checkArgument(EnumSet.of(MANAGED_TABLE, EXTERNAL_TABLE).contains(tableType), "Invalid table type: %s", tableType);
    try {
        Map<Path, byte[]> schemaFiles = new LinkedHashMap<>();
        for (PartitionWithStatistics partitionWithStatistics : partitions) {
            Partition partition = partitionWithStatistics.getPartition();
            verifiedPartition(table, partition);
            Path partitionMetadataDirectory = getPartitionMetadataDirectory(table, partition.getValues());
            Path schemaPath = getSchemaPath(PARTITION, partitionMetadataDirectory);
            if (metadataFileSystem.exists(schemaPath)) {
                throw new TrinoException(HIVE_METASTORE_ERROR, "Partition already exists");
            }
            byte[] schemaJson = partitionCodec.toJsonBytes(new PartitionMetadata(table, partitionWithStatistics));
            schemaFiles.put(schemaPath, schemaJson);
        }
        Set<Path> createdFiles = new LinkedHashSet<>();
        try {
            for (Entry<Path, byte[]> entry : schemaFiles.entrySet()) {
                try (OutputStream outputStream = metadataFileSystem.create(entry.getKey())) {
                    createdFiles.add(entry.getKey());
                    outputStream.write(entry.getValue());
                } catch (IOException e) {
                    throw new TrinoException(HIVE_METASTORE_ERROR, "Could not write partition schema", e);
                }
            }
        } catch (Throwable e) {
            for (Path createdFile : createdFiles) {
                try {
                    metadataFileSystem.delete(createdFile, false);
                } catch (IOException ignored) {
                }
            }
            throw e;
        }
    } catch (IOException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 3 with PartitionWithStatistics

use of io.trino.plugin.hive.metastore.PartitionWithStatistics in project trino by trinodb.

the class ThriftHiveMetastore method addPartitions.

@Override
public void addPartitions(HiveIdentity identity, String databaseName, String tableName, List<PartitionWithStatistics> partitionsWithStatistics) {
    List<Partition> partitions = partitionsWithStatistics.stream().map(ThriftMetastoreUtil::toMetastoreApiPartition).collect(toImmutableList());
    addPartitionsWithoutStatistics(identity, databaseName, tableName, partitions);
    for (PartitionWithStatistics partitionWithStatistics : partitionsWithStatistics) {
        storePartitionColumnStatistics(identity, databaseName, tableName, partitionWithStatistics.getPartitionName(), partitionWithStatistics);
    }
}

Example 4 with PartitionWithStatistics

use of io.trino.plugin.hive.metastore.PartitionWithStatistics in project trino by trinodb.

the class AbstractTestHive method createDummyPartitionedTable.

protected void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns) throws Exception {
    doCreateEmptyTable(tableName, ORC, columns);
    HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient());
    Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    List<String> firstPartitionValues = ImmutableList.of("2016-01-01");
    List<String> secondPartitionValues = ImmutableList.of("2016-01-02");
    String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues);
    String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues);
    List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName).stream().map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())).collect(toImmutableList());
    metastoreClient.addPartitions(tableName.getSchemaName(), tableName.getTableName(), partitions);
    metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
    metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
}

Example 5 with PartitionWithStatistics

use of io.trino.plugin.hive.metastore.PartitionWithStatistics in project trino by trinodb.

the class BaseTestHiveOnDataLake method renamePartitionResourcesOutsideTrino.

private void renamePartitionResourcesOutsideTrino(String tableName, String partitionColumn, String regionKey) {
    String partitionName = format("%s=%s", partitionColumn, regionKey);
    String partitionS3KeyPrefix = format("%s/%s/%s", HIVE_TEST_SCHEMA, tableName, partitionName);
    String renamedPartitionSuffix = "CP";
    // Copy whole partition to new location
    AmazonS3 amazonS3 = dockerizedS3DataLake.getS3Client();
    amazonS3.listObjects(bucketName).getObjectSummaries().forEach(object -> {
        String objectKey = object.getKey();
        if (objectKey.startsWith(partitionS3KeyPrefix)) {
            String fileName = objectKey.substring(objectKey.lastIndexOf('/'));
            String destinationKey = partitionS3KeyPrefix + renamedPartitionSuffix + fileName;
            amazonS3.copyObject(bucketName, objectKey, bucketName, destinationKey);
        }
    });
    // Delete old partition and update metadata to point to location of new copy
    Table hiveTable = metastoreClient.getTable(HIVE_TEST_SCHEMA, tableName).get();
    Partition hivePartition = metastoreClient.getPartition(hiveTable, List.of(regionKey)).get();
    Map<String, PartitionStatistics> partitionStatistics = metastoreClient.getPartitionStatistics(hiveTable, List.of(hivePartition));
    metastoreClient.dropPartition(HIVE_TEST_SCHEMA, tableName, List.of(regionKey), true);
    metastoreClient.addPartitions(HIVE_TEST_SCHEMA, tableName, List.of(new PartitionWithStatistics(Partition.builder(hivePartition).withStorage(builder -> builder.setLocation(hivePartition.getStorage().getLocation() + renamedPartitionSuffix)).build(), partitionName, partitionStatistics.get(partitionName))));
}