Examples with GluePartitionConverter io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.GluePartitionConverter used on opensource projects

Example 1 with GluePartitionConverter

use of io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.GluePartitionConverter in project trino by trinodb.

the class GlueHiveMetastore method batchGetPartition.

private List<Partition> batchGetPartition(Table table, List<String> partitionNames) {
    try {
        List<PartitionValueList> pendingPartitions = partitionNames.stream().map(partitionName -> new PartitionValueList().withValues(toPartitionValues(partitionName))).collect(toCollection(ArrayList::new));
        ImmutableList.Builder<Partition> resultsBuilder = ImmutableList.builderWithExpectedSize(partitionNames.size());
        // Reuse immutable field instances opportunistically between partitions
        GluePartitionConverter converter = new GluePartitionConverter(table);
        while (!pendingPartitions.isEmpty()) {
            List<Future<BatchGetPartitionResult>> batchGetPartitionFutures = new ArrayList<>();
            for (List<PartitionValueList> partitions : Lists.partition(pendingPartitions, BATCH_GET_PARTITION_MAX_PAGE_SIZE)) {
                long startTimestamp = System.currentTimeMillis();
                batchGetPartitionFutures.add(glueClient.batchGetPartitionAsync(new BatchGetPartitionRequest().withCatalogId(catalogId).withDatabaseName(table.getDatabaseName()).withTableName(table.getTableName()).withPartitionsToGet(partitions), new StatsRecordingAsyncHandler(stats.getGetPartitions(), startTimestamp)));
            }
            pendingPartitions.clear();
            for (Future<BatchGetPartitionResult> future : batchGetPartitionFutures) {
                BatchGetPartitionResult batchGetPartitionResult = future.get();
                List<com.amazonaws.services.glue.model.Partition> partitions = batchGetPartitionResult.getPartitions();
                List<PartitionValueList> unprocessedKeys = batchGetPartitionResult.getUnprocessedKeys();
                // In the unlikely scenario where batchGetPartition call cannot make progress on retrieving partitions, avoid infinite loop
                if (partitions.isEmpty()) {
                    verify(!unprocessedKeys.isEmpty(), "Empty unprocessedKeys for non-empty BatchGetPartitionRequest and empty partitions result");
                    throw new TrinoException(HIVE_METASTORE_ERROR, "Cannot make progress retrieving partitions. Unable to retrieve partitions: " + unprocessedKeys);
                }
                partitions.stream().map(converter).forEach(resultsBuilder::add);
                pendingPartitions.addAll(unprocessedKeys);
            }
        }
        return resultsBuilder.build();
    } catch (AmazonServiceException | InterruptedException | ExecutionException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 2 with GluePartitionConverter

use of io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.GluePartitionConverter in project trino by trinodb.

the class GlueHiveMetastore method getPartitions.

private List<Partition> getPartitions(Table table, String expression, @Nullable Segment segment) {
    try {
        // Reuse immutable field instances opportunistically between partitions
        GluePartitionConverter converter = new GluePartitionConverter(table);
        List<Partition> partitions = getPaginatedResults(glueClient::getPartitions, new GetPartitionsRequest().withCatalogId(catalogId).withDatabaseName(table.getDatabaseName()).withTableName(table.getTableName()).withExpression(expression).withSegment(segment).withMaxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS), GetPartitionsRequest::setNextToken, GetPartitionsResult::getNextToken, stats.getGetPartitions()).map(GetPartitionsResult::getPartitions).flatMap(List::stream).map(converter).collect(toImmutableList());
        return partitions;
    } catch (AmazonServiceException e) {
        throw new TrinoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 3 with GluePartitionConverter

use of io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.GluePartitionConverter in project trino by trinodb.

the class TestGlueToTrinoConverter method testConvertPartition.

@Test
public void testConvertPartition() {
    GluePartitionConverter converter = createPartitionConverter(testTable);
    io.trino.plugin.hive.metastore.Partition trinoPartition = converter.apply(testPartition);
    assertEquals(trinoPartition.getDatabaseName(), testPartition.getDatabaseName());
    assertEquals(trinoPartition.getTableName(), testPartition.getTableName());
    assertColumnList(trinoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns());
    assertEquals(trinoPartition.getValues(), testPartition.getValues());
    assertStorage(trinoPartition.getStorage(), testPartition.getStorageDescriptor());
    assertEquals(trinoPartition.getParameters(), testPartition.getParameters());
}

Example 4 with GluePartitionConverter

use of io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.GluePartitionConverter in project trino by trinodb.

the class TestGlueToTrinoConverter method testPartitionConversionMemoization.

@Test
public void testPartitionConversionMemoization() {
    String fakeS3Location = "s3://some-fake-location";
    testPartition.getStorageDescriptor().setLocation(fakeS3Location);
    // Second partition to convert with equal (but not aliased) values
    Partition partitionTwo = getGlueTestPartition("" + testDatabase.getName(), "" + testTable.getName(), new ArrayList<>(testPartition.getValues()));
    // Ensure storage fields match as well
    partitionTwo.getStorageDescriptor().setColumns(new ArrayList<>(testPartition.getStorageDescriptor().getColumns()));
    partitionTwo.getStorageDescriptor().setBucketColumns(new ArrayList<>(testPartition.getStorageDescriptor().getBucketColumns()));
    partitionTwo.getStorageDescriptor().setLocation("" + fakeS3Location);
    partitionTwo.getStorageDescriptor().setInputFormat("" + testPartition.getStorageDescriptor().getInputFormat());
    partitionTwo.getStorageDescriptor().setOutputFormat("" + testPartition.getStorageDescriptor().getOutputFormat());
    partitionTwo.getStorageDescriptor().setParameters(new HashMap<>(testPartition.getStorageDescriptor().getParameters()));
    GluePartitionConverter converter = createPartitionConverter(testTable);
    io.trino.plugin.hive.metastore.Partition trinoPartition = converter.apply(testPartition);
    io.trino.plugin.hive.metastore.Partition trinoPartition2 = converter.apply(partitionTwo);
    assertNotSame(trinoPartition, trinoPartition2);
    assertSame(trinoPartition2.getDatabaseName(), trinoPartition.getDatabaseName());
    assertSame(trinoPartition2.getTableName(), trinoPartition.getTableName());
    assertSame(trinoPartition2.getColumns(), trinoPartition.getColumns());
    assertSame(trinoPartition2.getParameters(), trinoPartition.getParameters());
    assertNotSame(trinoPartition2.getValues(), trinoPartition.getValues());
    Storage storage = trinoPartition.getStorage();
    Storage storage2 = trinoPartition2.getStorage();
    assertSame(storage2.getStorageFormat(), storage.getStorageFormat());
    assertSame(storage2.getBucketProperty(), storage.getBucketProperty());
    assertSame(storage2.getSerdeParameters(), storage.getSerdeParameters());
    assertNotSame(storage2.getLocation(), storage.getLocation());
}