Examples with GluePartitionConverter com.facebook.presto.hive.metastore.glue.converter.GlueToPrestoConverter.GluePartitionConverter used on opensource projects

Example 1 with GluePartitionConverter

use of com.facebook.presto.hive.metastore.glue.converter.GlueToPrestoConverter.GluePartitionConverter in project presto by prestodb.

the class TestGlueToPrestoConverter method testConvertPartition.

@Test
public void testConvertPartition() {
    GluePartitionConverter converter = new GluePartitionConverter(testPartition.getDatabaseName(), testPartition.getTableName());
    com.facebook.presto.hive.metastore.Partition prestoPartition = converter.apply(testPartition);
    assertEquals(prestoPartition.getDatabaseName(), testPartition.getDatabaseName());
    assertEquals(prestoPartition.getTableName(), testPartition.getTableName());
    assertColumnList(prestoPartition.getColumns(), testPartition.getStorageDescriptor().getColumns());
    assertEquals(prestoPartition.getValues(), testPartition.getValues());
    assertStorage(prestoPartition.getStorage(), testPartition.getStorageDescriptor());
    assertEquals(prestoPartition.getParameters(), testPartition.getParameters());
}

Example 2 with GluePartitionConverter

use of com.facebook.presto.hive.metastore.glue.converter.GlueToPrestoConverter.GluePartitionConverter in project presto by prestodb.

the class GlueHiveMetastore method batchGetPartition.

private List<Partition> batchGetPartition(String databaseName, String tableName, List<String> partitionNames) {
    try {
        List<Future<BatchGetPartitionResult>> batchGetPartitionFutures = new ArrayList<>();
        for (List<String> partitionNamesBatch : Lists.partition(partitionNames, BATCH_GET_PARTITION_MAX_PAGE_SIZE)) {
            List<PartitionValueList> partitionValuesBatch = mappedCopy(partitionNamesBatch, partitionName -> new PartitionValueList().withValues(toPartitionValues(partitionName)));
            batchGetPartitionFutures.add(glueClient.batchGetPartitionAsync(new BatchGetPartitionRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withPartitionsToGet(partitionValuesBatch), stats.getBatchGetPartitions().metricsAsyncHandler()));
        }
        GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName);
        ImmutableList.Builder<Partition> resultsBuilder = ImmutableList.builderWithExpectedSize(partitionNames.size());
        for (Future<BatchGetPartitionResult> future : batchGetPartitionFutures) {
            future.get().getPartitions().stream().map(converter).forEach(resultsBuilder::add);
        }
        return resultsBuilder.build();
    } catch (AmazonServiceException | InterruptedException | ExecutionException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 3 with GluePartitionConverter

use of com.facebook.presto.hive.metastore.glue.converter.GlueToPrestoConverter.GluePartitionConverter in project presto by prestodb.

the class GlueHiveMetastore method getPartitions.

private List<Partition> getPartitions(String databaseName, String tableName, String expression, @Nullable Segment segment) {
    try {
        GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName);
        ArrayList<Partition> partitions = new ArrayList<>();
        GetPartitionsRequest request = new GetPartitionsRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withExpression(expression).withSegment(segment).withMaxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS);
        do {
            GetPartitionsResult result = stats.getGetPartitions().record(() -> glueClient.getPartitions(request));
            request.setNextToken(result.getNextToken());
            partitions.ensureCapacity(partitions.size() + result.getPartitions().size());
            result.getPartitions().stream().map(converter).forEach(partitions::add);
        } while (request.getNextToken() != null);
        return partitions;
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}

Example 4 with GluePartitionConverter

use of com.facebook.presto.hive.metastore.glue.converter.GlueToPrestoConverter.GluePartitionConverter in project presto by prestodb.

the class TestGlueToPrestoConverter method testPartitionConversionMemoization.

@Test
public void testPartitionConversionMemoization() {
    String fakeS3Location = "s3://some-fake-location";
    testPartition.getStorageDescriptor().setLocation(fakeS3Location);
    // Second partition to convert with equal (but not aliased) values
    Partition partitionTwo = getGlueTestPartition(testPartition.getDatabaseName(), testPartition.getTableName(), new ArrayList<>(testPartition.getValues()));
    // Ensure storage fields are equal but not aliased as well
    partitionTwo.getStorageDescriptor().setColumns(new ArrayList<>(testPartition.getStorageDescriptor().getColumns()));
    partitionTwo.getStorageDescriptor().setBucketColumns(new ArrayList<>(testPartition.getStorageDescriptor().getBucketColumns()));
    partitionTwo.getStorageDescriptor().setLocation("" + fakeS3Location);
    partitionTwo.getStorageDescriptor().setInputFormat("" + testPartition.getStorageDescriptor().getInputFormat());
    partitionTwo.getStorageDescriptor().setOutputFormat("" + testPartition.getStorageDescriptor().getOutputFormat());
    partitionTwo.getStorageDescriptor().setParameters(new HashMap<>(testPartition.getStorageDescriptor().getParameters()));
    GluePartitionConverter converter = new GluePartitionConverter(testDb.getName(), testTbl.getName());
    com.facebook.presto.hive.metastore.Partition prestoPartition = converter.apply(testPartition);
    com.facebook.presto.hive.metastore.Partition prestoPartition2 = converter.apply(partitionTwo);
    assertNotSame(prestoPartition, prestoPartition2);
    assertSame(prestoPartition2.getDatabaseName(), prestoPartition.getDatabaseName());
    assertSame(prestoPartition2.getTableName(), prestoPartition.getTableName());
    assertSame(prestoPartition2.getColumns(), prestoPartition.getColumns());
    assertSame(prestoPartition2.getParameters(), prestoPartition.getParameters());
    assertNotSame(prestoPartition2.getValues(), prestoPartition.getValues());
    Storage storage = prestoPartition.getStorage();
    Storage storage2 = prestoPartition2.getStorage();
    assertSame(storage2.getStorageFormat(), storage.getStorageFormat());
    assertSame(storage2.getBucketProperty(), storage.getBucketProperty());
    assertSame(storage2.getSerdeParameters(), storage.getSerdeParameters());
    assertNotSame(storage2.getLocation(), storage.getLocation());
}

Example 5 with GluePartitionConverter

use of com.facebook.presto.hive.metastore.glue.converter.GlueToPrestoConverter.GluePartitionConverter in project presto by prestodb.

the class TestGlueToPrestoConverter method testPartitionNullParameters.

@Test
public void testPartitionNullParameters() {
    testPartition.setParameters(null);
    assertNotNull(new GluePartitionConverter(testDb.getName(), testTbl.getName()).apply(testPartition).getParameters());
}