use of com.amazonaws.services.glue.model.GetPartitionsResult in project alluxio by Alluxio.
the class GlueDatabase method batchGetPartitions.
private List<Partition> batchGetPartitions(AWSGlueAsync glueClient, String tableName) throws IOException {
// TODO(shouwei): make getPartition multi-thread to accelerate the large table fetching
List<Partition> partitions = new ArrayList<>();
String nextToken = null;
try {
do {
GetPartitionsRequest getPartitionsRequest = new GetPartitionsRequest().withCatalogId(mGlueConfiguration.get(Property.CATALOG_ID)).withDatabaseName(mGlueDbName).withTableName(tableName).withMaxResults(mGlueConfiguration.getInt(Property.MAX_GLUE_FETCH_PARTITIONS)).withNextToken(nextToken);
GetPartitionsResult getPartitionsResult = glueClient.getPartitions(getPartitionsRequest);
partitions.addAll(getPartitionsResult.getPartitions());
nextToken = getPartitionsResult.getNextToken();
LOG.debug("Glue table {}.{} adding {} batch partitions with total {} partitions.", mGlueDbName, tableName, getPartitionsResult.getPartitions().size(), partitions.size());
} while (nextToken != null);
if (partitions != null) {
LOG.info("Glue table {}.{} has {} partitions.", mGlueDbName, tableName, partitions.size());
if (LOG.isDebugEnabled()) {
partitions.stream().forEach(partition -> LOG.debug("Glue table {}.{} with partition: {}.", partition.getDatabaseName(), tableName, partition));
}
}
return partitions;
} catch (AWSGlueException e) {
throw new IOException("Cannot get partition information for table: " + tableName + " in Database: " + mGlueDbName + "; Catalog ID: " + mGlueConfiguration.get(Property.CATALOG_ID) + ". error: " + e.getMessage(), e);
}
}
use of com.amazonaws.services.glue.model.GetPartitionsResult in project presto by prestodb.
the class GlueHiveMetastore method getPartitions.
private List<Partition> getPartitions(String databaseName, String tableName, String expression, @Nullable Segment segment) {
try {
GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName);
ArrayList<Partition> partitions = new ArrayList<>();
GetPartitionsRequest request = new GetPartitionsRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withExpression(expression).withSegment(segment).withMaxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS);
do {
GetPartitionsResult result = stats.getGetPartitions().record(() -> glueClient.getPartitions(request));
request.setNextToken(result.getNextToken());
partitions.ensureCapacity(partitions.size() + result.getPartitions().size());
result.getPartitions().stream().map(converter).forEach(partitions::add);
} while (request.getNextToken() != null);
return partitions;
} catch (AmazonServiceException e) {
throw new PrestoException(HIVE_METASTORE_ERROR, e);
}
}
Aggregations