Examples with ShardInfo com.facebook.presto.raptor.metadata.ShardInfo used on opensource projects

Example 21 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class RaptorMetadata method finishDelete.

@Override
public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<Slice> fragments) {
    RaptorTableHandle table = (RaptorTableHandle) tableHandle;
    long transactionId = table.getTransactionId().getAsLong();
    long tableId = table.getTableId();
    List<ColumnInfo> columns = getColumnHandles(session, tableHandle).values().stream().map(RaptorColumnHandle.class::cast).map(ColumnInfo::fromHandle).collect(toList());
    if (table.isTableSupportsDeltaDelete()) {
        ImmutableMap.Builder<UUID, DeltaInfoPair> shardMapBuilder = ImmutableMap.builder();
        fragments.stream().map(fragment -> SHARD_DELETE_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> shardMapBuilder.put(delta.getOldShardUuid(), delta.getDeltaInfoPair()));
        OptionalLong updateTime = OptionalLong.of(session.getStartTime());
        log.info("Finishing delete for tableId %s (affected shardUuid: %s)", tableId, shardMapBuilder.build().size());
        shardManager.replaceDeltaUuids(transactionId, tableId, columns, shardMapBuilder.build(), updateTime);
    } else {
        ImmutableSet.Builder<UUID> oldShardUuidsBuilder = ImmutableSet.builder();
        ImmutableList.Builder<ShardInfo> newShardsBuilder = ImmutableList.builder();
        fragments.stream().map(fragment -> SHARD_DELTA_CODEC.fromJson(fragment.getBytes())).forEach(delta -> {
            oldShardUuidsBuilder.addAll(delta.getOldShardUuids());
            newShardsBuilder.addAll(delta.getNewShards());
        });
        Set<UUID> oldShardUuids = oldShardUuidsBuilder.build();
        List<ShardInfo> newShards = newShardsBuilder.build();
        OptionalLong updateTime = OptionalLong.of(session.getStartTime());
        log.info("Finishing delete for tableId %s (removed: %s, rewritten: %s)", tableId, oldShardUuids.size() - newShards.size(), newShards.size());
        shardManager.replaceShardUuids(transactionId, tableId, columns, oldShardUuids, newShards, updateTime);
    }
    clearRollback();
}

Example 22 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class TestOrcStorageManager method testWriteDeltaDeleteMerge.

@Test
public // rowsToDelete and rowsDeleted must be mutually exclusive
void testWriteDeltaDeleteMerge() throws Exception {
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    Collection<Slice> fragments = deltaDelete(rowsToDelete, true);
    Slice shardDelta = Iterables.getOnlyElement(fragments);
    ShardDeleteDelta shardDeltas = jsonCodec(ShardDeleteDelta.class).fromJson(shardDelta.getBytes());
    ShardInfo shardInfo = shardDeltas.getDeltaInfoPair().getNewDeltaDeleteShard().get();
    // Check that output file (new delta file) has merged 2 rows
    assertEquals(shardInfo.getRowCount(), 2);
    assertTrue(checkContent(fileSystem, shardInfo.getShardUuid(), rowsToDelete));
    // Check that storage file is same as backup file
    File storageFile = new File(storageService.getStorageFile(shardInfo.getShardUuid()).toString());
    File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid());
    assertFileEquals(storageFile, backupFile);
    // Verify recorded shard
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    // original file + old delta + new delta
    assertEquals(recordedShards.size(), 3);
    assertEquals(recordedShards.get(2).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(2).getShardUuid(), shardInfo.getShardUuid());
}

Example 23 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class TestOrcStorageManager method testWriter.

@Test
public void testWriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    // shard is not recorded until flush
    assertEquals(shardRecorder.getShards().size(), 0);
    sink.flush();
    // shard is recorded after flush
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 1);
    List<ShardInfo> shards = getFutureValue(sink.commit());
    assertEquals(shards.size(), 1);
    ShardInfo shardInfo = Iterables.getOnlyElement(shards);
    UUID shardUuid = shardInfo.getShardUuid();
    File file = new File(storageService.getStorageFile(shardUuid).toString());
    File backupFile = fileBackupStore.getBackupFile(shardUuid);
    assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(0).getShardUuid(), shardUuid);
    assertEquals(shardInfo.getRowCount(), 2);
    assertEquals(shardInfo.getCompressedSize(), file.length());
    assertEquals(shardInfo.getXxhash64(), xxhash64(new RaptorLocalFileSystem(new Configuration()), new Path(file.toURI())));
    // verify primary and backup shard exist
    assertFile(file, "primary shard");
    assertFile(backupFile, "backup shard");
    assertFileEquals(file, backupFile);
    // remove primary shard to force recovery from backup
    assertTrue(file.delete());
    assertTrue(file.getParentFile().delete());
    assertFalse(file.exists());
    recoveryManager.restoreFromBackup(shardUuid, shardInfo.getCompressedSize(), OptionalLong.of(shardInfo.getXxhash64()));
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    try (OrcDataSource dataSource = manager.openShard(fileSystem, shardUuid, READER_ATTRIBUTES)) {
        OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.nextBatch(), 2);
        Block column0 = reader.readBlock(0);
        assertEquals(column0.isNull(0), false);
        assertEquals(column0.isNull(1), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 456L);
        Block column1 = reader.readBlock(1);
        assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye"));
        assertEquals(reader.nextBatch(), -1);
    }
}

Example 24 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class TestShardCompactor method testShardCompactorWithDelta.

@Test
public void testShardCompactorWithDelta() throws Exception {
    StorageManager storageManager = createOrcStorageManager(dbi, temporary, MAX_SHARD_ROWS);
    List<Long> columnIds = ImmutableList.of(3L, 7L, 2L, 1L, 5L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE, DATE, TIMESTAMP);
    List<ShardInfo> inputShards = createShards(storageManager, columnIds, columnTypes, 3);
    assertEquals(inputShards.size(), 3);
    List<Long> deltaColumnIds = ImmutableList.of(1L);
    List<Type> deltaColumnTypes = ImmutableList.of(BIGINT);
    StoragePageSink deltaSink = createStoragePageSink(storageManager, deltaColumnIds, deltaColumnTypes);
    List<Page> deltaPages = rowPagesBuilder(deltaColumnTypes).row(1L).row(2L).build();
    deltaSink.appendPages(deltaPages);
    List<ShardInfo> deltaShards = getFutureValue(deltaSink.commit());
    long totalRows = inputShards.stream().mapToLong(ShardInfo::getRowCount).sum();
    long expectedOutputShardsCount = computeExpectedOutputShards(totalRows - 2);
    Map<UUID, Optional<UUID>> inputUuidsMap = new HashMap<>();
    inputUuidsMap.put(inputShards.get(0).getShardUuid(), Optional.of(deltaShards.get(0).getShardUuid()));
    inputUuidsMap.put(inputShards.get(1).getShardUuid(), Optional.empty());
    inputUuidsMap.put(inputShards.get(2).getShardUuid(), Optional.empty());
    long transactionId = 1;
    ShardCompactor compactor = new ShardCompactor(storageManager, READER_ATTRIBUTES);
    List<ShardInfo> outputShards = compactor.compact(transactionId, true, OptionalInt.empty(), inputUuidsMap, getColumnInfo(columnIds, columnTypes));
    assertEquals(outputShards.size(), expectedOutputShardsCount);
    Set<UUID> outputUuids = outputShards.stream().map(ShardInfo::getShardUuid).collect(toSet());
    assertShardEqualsIgnoreOrder(storageManager, inputUuidsMap, outputUuids, columnIds, columnTypes);
}

Example 25 with ShardInfo

use of com.facebook.presto.raptor.metadata.ShardInfo in project presto by prestodb.

the class InplaceShardRewriter method rewriteShard.

@VisibleForTesting
Collection<Slice> rewriteShard(BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    UUID newShardUuid = UUID.randomUUID();
    Path input = storageService.getStorageFile(shardUuid);
    Path output = storageService.getStagingFile(newShardUuid);
    OrcFileInfo info = orcStorageManager.rewriteFile(fileSystem, columns, input, output, rowsToDelete);
    long rowCount = info.getRowCount();
    if (rowCount == 0) {
        return shardDelta(shardUuid, Optional.empty());
    }
    shardRecorder.recordCreatedShard(transactionId, newShardUuid);
    // submit for backup and wait until it finishes
    getFutureValue(backupManager.submit(newShardUuid, output));
    Set<String> nodes = ImmutableSet.of(nodeId);
    long uncompressedSize = info.getUncompressedSize();
    ShardInfo shard = orcStorageManager.createShardInfo(fileSystem, newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize);
    orcStorageManager.writeShard(newShardUuid);
    return shardDelta(shardUuid, Optional.of(shard));
}