Examples with StoragePageSink com.facebook.presto.raptor.storage.StoragePageSink used on opensource projects

Example 1 with StoragePageSink

use of com.facebook.presto.raptor.storage.StoragePageSink in project presto by prestodb.

the class TestShardCompactor method createShards.

private static List<ShardInfo> createShards(StorageManager storageManager, List<Long> columnIds, List<Type> columnTypes, int shardCount) {
    StoragePageSink sink = createStoragePageSink(storageManager, columnIds, columnTypes);
    for (int i = 0; i < shardCount; i++) {
        sink.appendPages(createPages(columnTypes));
        sink.flush();
    }
    return getFutureValue(sink.commit());
}

Example 2 with StoragePageSink

use of com.facebook.presto.raptor.storage.StoragePageSink in project presto by prestodb.

the class ShardCompactor method compact.

public List<ShardInfo> compact(long transactionId, boolean tableSupportsDeltaDelete, OptionalInt bucketNumber, Map<UUID, Optional<UUID>> uuidsMap, List<ColumnInfo> columns) throws IOException {
    long start = System.nanoTime();
    List<Long> columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList());
    List<Type> columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList());
    StoragePageSink storagePageSink = storageManager.createStoragePageSink(DEFAULT_RAPTOR_CONTEXT, transactionId, bucketNumber, columnIds, columnTypes, false);
    List<ShardInfo> shardInfos;
    try {
        shardInfos = compact(storagePageSink, tableSupportsDeltaDelete, bucketNumber, uuidsMap, columnIds, columnTypes);
    } catch (IOException | RuntimeException e) {
        storagePageSink.rollback();
        throw e;
    }
    int deltaCount = uuidsMap.values().stream().filter(Optional::isPresent).collect(toSet()).size();
    updateStats(uuidsMap.size(), deltaCount, shardInfos.size(), nanosSince(start).toMillis());
    return shardInfos;
}

Example 3 with StoragePageSink

use of com.facebook.presto.raptor.storage.StoragePageSink in project presto by prestodb.

the class ShardCompactor method compactSorted.

public List<ShardInfo> compactSorted(long transactionId, OptionalInt bucketNumber, Set<UUID> uuids, List<ColumnInfo> columns, List<Long> sortColumnIds, List<SortOrder> sortOrders) throws IOException {
    checkArgument(sortColumnIds.size() == sortOrders.size(), "sortColumnIds and sortOrders must be of the same size");
    long start = System.nanoTime();
    List<Long> columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList());
    List<Type> columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList());
    checkArgument(columnIds.containsAll(sortColumnIds), "sortColumnIds must be a subset of columnIds");
    List<Integer> sortIndexes = sortColumnIds.stream().map(columnIds::indexOf).collect(toList());
    Queue<SortedRowSource> rowSources = new PriorityQueue<>();
    StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false);
    try {
        for (UUID uuid : uuids) {
            ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes);
            SortedRowSource rowSource = new SortedRowSource(pageSource, columnTypes, sortIndexes, sortOrders);
            rowSources.add(rowSource);
        }
        while (!rowSources.isEmpty()) {
            SortedRowSource rowSource = rowSources.poll();
            if (!rowSource.hasNext()) {
                // rowSource is empty, close it
                rowSource.close();
                continue;
            }
            outputPageSink.appendRow(rowSource.next());
            if (outputPageSink.isFull()) {
                outputPageSink.flush();
            }
            rowSources.add(rowSource);
        }
        outputPageSink.flush();
        List<ShardInfo> shardInfos = getFutureValue(outputPageSink.commit());
        updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis());
        return shardInfos;
    } catch (IOException | RuntimeException e) {
        outputPageSink.rollback();
        throw e;
    } finally {
        rowSources.forEach(SortedRowSource::closeQuietly);
    }
}

Example 4 with StoragePageSink

use of com.facebook.presto.raptor.storage.StoragePageSink in project presto by prestodb.

the class ShardCompactor method compact.

public List<ShardInfo> compact(long transactionId, OptionalInt bucketNumber, Set<UUID> uuids, List<ColumnInfo> columns) throws IOException {
    long start = System.nanoTime();
    List<Long> columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList());
    List<Type> columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList());
    StoragePageSink storagePageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false);
    List<ShardInfo> shardInfos;
    try {
        shardInfos = compact(storagePageSink, bucketNumber, uuids, columnIds, columnTypes);
    } catch (IOException | RuntimeException e) {
        storagePageSink.rollback();
        throw e;
    }
    updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis());
    return shardInfos;
}

Example 5 with StoragePageSink

use of com.facebook.presto.raptor.storage.StoragePageSink in project presto by prestodb.

the class ShardCompactor method compactSorted.

public List<ShardInfo> compactSorted(long transactionId, boolean tableSupportsDeltaDelete, OptionalInt bucketNumber, Map<UUID, Optional<UUID>> uuidsMap, List<ColumnInfo> columns, List<Long> sortColumnIds, List<SortOrder> sortOrders) throws IOException {
    checkArgument(sortColumnIds.size() == sortOrders.size(), "sortColumnIds and sortOrders must be of the same size");
    long start = System.nanoTime();
    List<Long> columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList());
    List<Type> columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList());
    checkArgument(columnIds.containsAll(sortColumnIds), "sortColumnIds must be a subset of columnIds");
    List<Integer> sortIndexes = sortColumnIds.stream().map(columnIds::indexOf).collect(toList());
    Queue<SortedPageSource> rowSources = new PriorityQueue<>();
    StoragePageSink outputPageSink = storageManager.createStoragePageSink(DEFAULT_RAPTOR_CONTEXT, transactionId, bucketNumber, columnIds, columnTypes, false);
    try {
        uuidsMap.forEach((uuid, deltaUuid) -> {
            ConnectorPageSource pageSource = storageManager.getPageSource(DEFAULT_RAPTOR_CONTEXT, DEFAULT_HIVE_FILE_CONTEXT, uuid, deltaUuid, tableSupportsDeltaDelete, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes);
            SortedPageSource rowSource = new SortedPageSource(pageSource, columnTypes, sortIndexes, sortOrders);
            rowSources.add(rowSource);
        });
        while (!rowSources.isEmpty()) {
            SortedPageSource rowSource = rowSources.poll();
            if (!rowSource.hasNext()) {
                // rowSource is empty, close it
                rowSource.close();
                continue;
            }
            outputPageSink.appendPages(ImmutableList.of(rowSource.next()));
            if (outputPageSink.isFull()) {
                outputPageSink.flush();
            }
            rowSources.add(rowSource);
        }
        outputPageSink.flush();
        List<ShardInfo> shardInfos = getFutureValue(outputPageSink.commit());
        int deltaCount = uuidsMap.values().stream().filter(Optional::isPresent).collect(toSet()).size();
        updateStats(uuidsMap.size(), deltaCount, shardInfos.size(), nanosSince(start).toMillis());
        return shardInfos;
    } catch (IOException | RuntimeException e) {
        outputPageSink.rollback();
        throw e;
    } finally {
        rowSources.forEach(SortedPageSource::closeQuietly);
    }
}