Examples with RecordedShard com.facebook.presto.raptor.storage.InMemoryShardRecorder.RecordedShard used on opensource projects

Example 1 with RecordedShard

use of com.facebook.presto.raptor.storage.InMemoryShardRecorder.RecordedShard in project presto by prestodb.

the class TestOrcStorageManager method testWriter.

@Test
public void testWriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    // shard is not recorded until flush
    assertEquals(shardRecorder.getShards().size(), 0);
    sink.flush();
    // shard is recorded after flush
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 1);
    List<ShardInfo> shards = getFutureValue(sink.commit());
    assertEquals(shards.size(), 1);
    ShardInfo shardInfo = Iterables.getOnlyElement(shards);
    UUID shardUuid = shardInfo.getShardUuid();
    File file = storageService.getStorageFile(shardUuid);
    File backupFile = fileBackupStore.getBackupFile(shardUuid);
    assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(0).getShardUuid(), shardUuid);
    assertEquals(shardInfo.getRowCount(), 2);
    assertEquals(shardInfo.getCompressedSize(), file.length());
    // verify primary and backup shard exist
    assertFile(file, "primary shard");
    assertFile(backupFile, "backup shard");
    assertFileEquals(file, backupFile);
    // remove primary shard to force recovery from backup
    assertTrue(file.delete());
    assertTrue(file.getParentFile().delete());
    assertFalse(file.exists());
    recoveryManager.restoreFromBackup(shardUuid, OptionalLong.empty());
    try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_ATTRIBUTES)) {
        OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.nextBatch(), 2);
        Block column0 = reader.readBlock(BIGINT, 0);
        assertEquals(column0.isNull(0), false);
        assertEquals(column0.isNull(1), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 456L);
        Block column1 = reader.readBlock(createVarcharType(10), 1);
        assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye"));
        assertEquals(reader.nextBatch(), -1);
    }
}

Example 2 with RecordedShard

use of com.facebook.presto.raptor.storage.InMemoryShardRecorder.RecordedShard in project presto by prestodb.

the class TestOrcStorageManager method testRewriter.

@Test
public void testRewriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();
    long transactionId = TRANSACTION_ID;
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10));
    // create file with 2 rows
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    List<ShardInfo> shards = getFutureValue(sink.commit());
    assertEquals(shardRecorder.getShards().size(), 1);
    // delete one row
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    Collection<Slice> fragments = manager.rewriteShard(transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete);
    Slice shardDelta = Iterables.getOnlyElement(fragments);
    ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes());
    ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards());
    // check that output file has one row
    assertEquals(shardInfo.getRowCount(), 1);
    // check that storage file is same as backup file
    File storageFile = storageService.getStorageFile(shardInfo.getShardUuid());
    File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid());
    assertFileEquals(storageFile, backupFile);
    // verify recorded shard
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 2);
    assertEquals(recordedShards.get(1).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid());
}