Examples with GetPartitionsRequest com.amazonaws.services.glue.model.GetPartitionsRequest used on opensource projects

Example 1 with GetPartitionsRequest

use of com.amazonaws.services.glue.model.GetPartitionsRequest in project alluxio by Alluxio.

the class GlueDatabase method batchGetPartitions.

private List<Partition> batchGetPartitions(AWSGlueAsync glueClient, String tableName) throws IOException {
    // TODO(shouwei): make getPartition multi-thread to accelerate the large table fetching
    List<Partition> partitions = new ArrayList<>();
    String nextToken = null;
    try {
        do {
            GetPartitionsRequest getPartitionsRequest = new GetPartitionsRequest().withCatalogId(mGlueConfiguration.get(Property.CATALOG_ID)).withDatabaseName(mGlueDbName).withTableName(tableName).withMaxResults(mGlueConfiguration.getInt(Property.MAX_GLUE_FETCH_PARTITIONS)).withNextToken(nextToken);
            GetPartitionsResult getPartitionsResult = glueClient.getPartitions(getPartitionsRequest);
            partitions.addAll(getPartitionsResult.getPartitions());
            nextToken = getPartitionsResult.getNextToken();
            LOG.debug("Glue table {}.{} adding {} batch partitions with total {} partitions.", mGlueDbName, tableName, getPartitionsResult.getPartitions().size(), partitions.size());
        } while (nextToken != null);
        if (partitions != null) {
            LOG.info("Glue table {}.{} has {} partitions.", mGlueDbName, tableName, partitions.size());
            if (LOG.isDebugEnabled()) {
                partitions.stream().forEach(partition -> LOG.debug("Glue table {}.{} with partition: {}.", partition.getDatabaseName(), tableName, partition));
            }
        }
        return partitions;
    } catch (AWSGlueException e) {
        throw new IOException("Cannot get partition information for table: " + tableName + " in Database: " + mGlueDbName + "; Catalog ID: " + mGlueConfiguration.get(Property.CATALOG_ID) + ". error: " + e.getMessage(), e);
    }
}

Example 2 with GetPartitionsRequest

use of com.amazonaws.services.glue.model.GetPartitionsRequest in project presto by prestodb.

the class GlueHiveMetastore method getPartitions.

private List<Partition> getPartitions(String databaseName, String tableName, String expression, @Nullable Segment segment) {
    try {
        GluePartitionConverter converter = new GluePartitionConverter(databaseName, tableName);
        ArrayList<Partition> partitions = new ArrayList<>();
        GetPartitionsRequest request = new GetPartitionsRequest().withCatalogId(catalogId).withDatabaseName(databaseName).withTableName(tableName).withExpression(expression).withSegment(segment).withMaxResults(AWS_GLUE_GET_PARTITIONS_MAX_RESULTS);
        do {
            GetPartitionsResult result = stats.getGetPartitions().record(() -> glueClient.getPartitions(request));
            request.setNextToken(result.getNextToken());
            partitions.ensureCapacity(partitions.size() + result.getPartitions().size());
            result.getPartitions().stream().map(converter).forEach(partitions::add);
        } while (request.getNextToken() != null);
        return partitions;
    } catch (AmazonServiceException e) {
        throw new PrestoException(HIVE_METASTORE_ERROR, e);
    }
}