Examples with OrcFileInfo com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo used on opensource projects

Example 1 with OrcFileInfo

use of com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo in project presto by prestodb.

the class OrcStorageManager method rewriteShard.

@VisibleForTesting
Collection<Slice> rewriteShard(long transactionId, OptionalInt bucketNumber, UUID shardUuid, BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    UUID newShardUuid = UUID.randomUUID();
    File input = storageService.getStorageFile(shardUuid);
    File output = storageService.getStagingFile(newShardUuid);
    OrcFileInfo info = rewriteFile(input, output, rowsToDelete);
    long rowCount = info.getRowCount();
    if (rowCount == 0) {
        return shardDelta(shardUuid, Optional.empty());
    }
    shardRecorder.recordCreatedShard(transactionId, newShardUuid);
    // submit for backup and wait until it finishes
    getFutureValue(backupManager.submit(newShardUuid, output));
    Set<String> nodes = ImmutableSet.of(nodeId);
    long uncompressedSize = info.getUncompressedSize();
    ShardInfo shard = createShardInfo(newShardUuid, bucketNumber, output, nodes, rowCount, uncompressedSize);
    writeShard(newShardUuid);
    return shardDelta(shardUuid, Optional.of(shard));
}

Example 2 with OrcFileInfo

use of com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo in project presto by prestodb.

the class TestOrcFileRewriter method testRewriteNoRowsDeleted.

@Test
public void testRewriteNoRowsDeleted() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L);
    List<Type> columnTypes = ImmutableList.of(BIGINT);
    File file = new File(temporary, randomUUID().toString());
    try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
        writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build());
    }
    BitSet rowsToDelete = new BitSet();
    File newFile = new File(temporary, randomUUID().toString());
    OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete);
    assertEquals(info.getRowCount(), 2);
    assertEquals(info.getUncompressedSize(), 16);
    assertEquals(readAllBytes(newFile.toPath()), readAllBytes(file.toPath()));
}

Example 3 with OrcFileInfo

use of com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo in project presto by prestodb.

the class TestOrcFileRewriter method testUncompressedSize.

@Test
public void testUncompressedSize() throws Exception {
    List<Long> columnIds = ImmutableList.of(1L, 2L, 3L, 4L, 5L);
    List<Type> columnTypes = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, createVarcharType(10), VARBINARY);
    File file = new File(temporary, randomUUID().toString());
    try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
        List<Page> pages = rowPagesBuilder(columnTypes).row(true, 123L, 98.7, "hello", utf8Slice("abc")).row(false, 456L, 65.4, "world", utf8Slice("xyz")).row(null, null, null, null, null).build();
        writer.appendPages(pages);
    }
    File newFile = new File(temporary, randomUUID().toString());
    OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, new BitSet());
    assertEquals(info.getRowCount(), 3);
    assertEquals(info.getUncompressedSize(), 55);
}

Example 4 with OrcFileInfo

use of com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo in project presto by prestodb.

the class TestOrcFileRewriter method testRewriteWithoutMetadata.

@Test
public void testRewriteWithoutMetadata() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20));
    File file = new File(temporary, randomUUID().toString());
    try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file, false)) {
        List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(777L, "sky").build();
        writer.appendPages(pages);
    }
    try (OrcDataSource dataSource = fileOrcDataSource(file)) {
        OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 2);
        assertEquals(reader.getFileRowCount(), 2);
        assertEquals(reader.getSplitLength(), file.length());
        assertEquals(reader.nextBatch(), 2);
        Block column0 = reader.readBlock(BIGINT, 0);
        assertEquals(column0.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column0.isNull(i), false);
        }
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 1), 777L);
        Block column1 = reader.readBlock(createVarcharType(20), 1);
        assertEquals(column1.getPositionCount(), 2);
        for (int i = 0; i < 2; i++) {
            assertEquals(column1.isNull(i), false);
        }
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(20).getSlice(column1, 1), utf8Slice("sky"));
        assertFalse(reader.getUserMetadata().containsKey(OrcFileMetadata.KEY));
    }
    BitSet rowsToDelete = new BitSet(5);
    rowsToDelete.set(1);
    File newFile = new File(temporary, randomUUID().toString());
    OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete);
    assertEquals(info.getRowCount(), 1);
    assertEquals(info.getUncompressedSize(), 13);
    try (OrcDataSource dataSource = fileOrcDataSource(newFile)) {
        OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 1);
        assertEquals(reader.getFileRowCount(), 1);
        assertEquals(reader.getSplitLength(), newFile.length());
        assertEquals(reader.nextBatch(), 1);
        Block column0 = reader.readBlock(BIGINT, 0);
        assertEquals(column0.getPositionCount(), 1);
        assertEquals(column0.isNull(0), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        Block column1 = reader.readBlock(createVarcharType(20), 1);
        assertEquals(column1.getPositionCount(), 1);
        assertEquals(column1.isNull(0), false);
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("hello"));
        assertFalse(reader.getUserMetadata().containsKey(OrcFileMetadata.KEY));
    }
}

Example 5 with OrcFileInfo

use of com.facebook.presto.raptor.storage.OrcFileRewriter.OrcFileInfo in project presto by prestodb.

the class TestOrcFileRewriter method testRewriteAllRowsDeleted.

@Test
public void testRewriteAllRowsDeleted() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L);
    List<Type> columnTypes = ImmutableList.of(BIGINT);
    File file = new File(temporary, randomUUID().toString());
    try (OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
        writer.appendPages(rowPagesBuilder(columnTypes).row(123L).row(456L).build());
    }
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    rowsToDelete.set(1);
    File newFile = new File(temporary, randomUUID().toString());
    OrcFileInfo info = OrcFileRewriter.rewrite(file, newFile, rowsToDelete);
    assertEquals(info.getRowCount(), 0);
    assertEquals(info.getUncompressedSize(), 0);
    assertFalse(newFile.exists());
}