Examples with ShardInfo com.facebook.presto.raptor.metadata.ShardInfo used on opensource projects

Example 1 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class OrcStorageManager method shardDelta.

private static Collection<Slice> shardDelta(UUID oldShardUuid, Optional<ShardInfo> shardInfo) {
    List<ShardInfo> newShards = shardInfo.map(ImmutableList::of).orElse(ImmutableList.of());
    ShardDelta delta = new ShardDelta(ImmutableList.of(oldShardUuid), newShards);
    return ImmutableList.of(Slices.wrappedBuffer(SHARD_DELTA_CODEC.toJsonBytes(delta)));
}

Example 2 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class OrcStorageManager method rewriteShard.

@VisibleForTesting
Collection<Slice> rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    UUID newShardUuid = UUID.randomUUID();
    File input = storageService.getStorageFile(shardUuid);
    File output = storageService.getStagingFile(newShardUuid);
    OrcFileInfo info = rewriteFile(input, output, rowsToDelete);
    long rowCount = info.getRowCount();
    if (rowCount == 0) {
        return shardDelta(shardUuid, Optional.empty());
    }
    shardRecorder.recordCreatedShard(transactionId, newShardUuid);
    // submit for backup and wait until it finishes
    getFutureValue(backupManager.submit(newShardUuid, output));
    Set<String> nodes = ImmutableSet.of(nodeId);
    long uncompressedSize = info.getUncompressedSize();
    ShardInfo shard = createShardInfo(newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize);
    writeShard(newShardUuid);
    return shardDelta(shardUuid, Optional.of(shard));
}

Example 3 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class OrganizationJob method runJob.

private void runJob(long transactionId, OptionalInt bucketNumber, long tableId, Set<UUID> shardUuids) throws IOException {
    TableMetadata metadata = getTableMetadata(tableId);
    List<ShardInfo> newShards = performCompaction(transactionId, bucketNumber, shardUuids, metadata);
    log.info("Compacted shards %s into %s", shardUuids, newShards.stream().map(ShardInfo::getShardUuid).collect(toList()));
    shardManager.replaceShardUuids(transactionId, tableId, metadata.getColumns(), shardUuids, newShards, OptionalLong.empty());
}

Example 4 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class TestShardCompactor method testShardCompactorSorted.

@Test(dataProvider = "useOptimizedOrcWriter")
public void testShardCompactorSorted(boolean useOptimizedOrcWriter) throws Exception {
    StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DATE, TIMESTAMP, DOUBLE);
    List<Long> columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L);
    List<Long> sortColumnIds = ImmutableList.of(1L, 2L, 3L, 5L, 7L);
    List<SortOrder> sortOrders = nCopies(sortColumnIds.size(), ASC_NULLS_FIRST);
    List<Integer> sortIndexes = sortColumnIds.stream().map(columnIds::indexOf).collect(toList());
    List<ShardInfo> inputShards = createSortedShards(storageManager, columnIds, columnTypes, sortIndexes, sortOrders, 2);
    assertEquals(inputShards.size(), 2);
    long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum();
    long expectedOutputShards = computeExpectedOutputShards(totalRows);
    Map<UUID, Optional<UUID>> inputUuids = new HashMap<>();
    for (ShardInfo shardInfo : inputShards) {
        inputUuids.put(shardInfo.getShardUuid(), Optional.empty());
    }
    long transactionId = 1;
    ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES);
    List<ShardInfo> outputShards = compactor.compactSorted(transactionId, false, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes), sortColumnIds, sortOrders);
    List<UUID> outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toList());
    assertEquals(outputShards.size(), expectedOutputShards);
    assertShardEqualsSorted(storageManager, inputUuids.keySet(), outputUuids, columnIds, columnTypes, sortIndexes, sortOrders);
}

Example 5 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class TestShardCompactor method testShardCompactor.

@Test(dataProvider = "useOptimizedOrcWriter")
public void testShardCompactor(boolean useOptimizedOrcWriter) throws Exception {
    StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS);
    List<Long> columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP);
    List<ShardInfo> inputShards = createShards(storageManager, columnIds, columnTypes, 3);
    assertEquals(inputShards.size(), 3);
    long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum();
    long expectedOutputShards = computeExpectedOutputShards(totalRows);
    Map<UUID, Optional<UUID>> inputUuids = new HashMap<>();
    for (ShardInfo shardInfo : inputShards) {
        inputUuids.put(shardInfo.getShardUuid(), Optional.empty());
    }
    long transactionId = 1;
    ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES);
    List<ShardInfo> outputShards = compactor.compact(transactionId, false, OptionalInt.empty(), inputUuids, getColumnInfo(columnIds, columnTypes));
    assertEquals(outputShards.size(), expectedOutputShards);
    Set<UUID> outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet());
    assertShardEqualsIgnoreOrder(storageManager, inputUuids.keySet(), outputUuids, columnIds, columnTypes);
}