Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 71 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestFilterFunction method assertFilter.

private static void assertFilter(FilterFunction filter, Block input, int[] inputPositions, int positionCount) {
    // Copy the positions array because filter mutates it.
    int[] positions = Arrays.copyOf(inputPositions, positionCount);
    RuntimeException[] errors = new RuntimeException[inputPositions[positionCount - 1] + 1];
    // Put a pre-existing error in the 1st half of the input.
    int numPreviousErrors = positionCount / 2;
    for (int i = 0; i < numPreviousErrors; i++) {
        errors[i] = new RuntimeException("Pre-existent error at  " + positions[i]);
    }
    int lastErrorPosition = numPreviousErrors > 0 ? positions[numPreviousErrors - 1] : -1;
    int numHits = filter.filter(new Page(positionCount, input), positions, positionCount, errors);
    int hitCounter = 0;
    for (int position : inputPositions) {
        long number = input.getLong(position);
        if (number == UNLUCKY) {
            assertEquals(positions[hitCounter], position);
            assertTrue(errors[hitCounter] instanceof UnluckyError);
            hitCounter++;
        } else if ((number & 1) == 1) {
            assertEquals(positions[hitCounter], position);
            if (position <= lastErrorPosition) {
                assertTrue(errors[hitCounter] instanceof RuntimeException);
            } else {
                assertEquals(errors[hitCounter], null);
            }
            hitCounter++;
        }
    }
    assertEquals(numHits, hitCounter);
}

Example 72 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class DeltaShardRewriter method writeDeltaDeleteFile.

@VisibleForTesting
Collection<Slice> writeDeltaDeleteFile(BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    // TODO: Under current implementation, one block can only hold INT_MAX many rows
    // which theoretically may not be enough to hold all rows from an ORC file.
    // At this point, rowsToDelete couldn't be empty
    oldDeltaDeleteShardUuid.ifPresent(oldDeltaDeleteShardUuid -> mergeToRowsToDelete(rowsToDelete, oldDeltaDeleteShardUuid));
    if (rowsToDelete.cardinality() == oldShardRowCount) {
        // Delete original file
        return shardDeleteDelta(oldShardUuid, oldDeltaDeleteShardUuid, Optional.empty());
    }
    BlockBuilder blockBuilder = new LongArrayBlockBuilder(null, rowsToDelete.size());
    for (int i = rowsToDelete.nextSetBit(0); i >= 0; i = rowsToDelete.nextSetBit(i + 1)) {
        blockBuilder.writeLong(i);
    }
    // blockToDelete is LongArrayBlock
    StoragePageSink pageSink = orcStorageManager.createStoragePageSink(hdfsContext, transactionId, bucketNumber, ImmutableList.of(0L), ImmutableList.of(BIGINT), true);
    pageSink.appendPages(ImmutableList.of(new Page(blockBuilder.build())));
    List<ShardInfo> shardInfos = getFutureValue(pageSink.commit());
    // Guaranteed that shardInfos only has one element since we only call commit one time
    ShardInfo newDeltaDeleteShard = Iterables.getOnlyElement(shardInfos);
    return shardDeleteDelta(oldShardUuid, oldDeltaDeleteShardUuid, Optional.of(newDeltaDeleteShard));
}

Example 73 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class OrcFileRewriter method rewrite.

private static OrcFileInfo rewrite(OrcBatchRecordReader reader, OrcWriter writer, BitSet rowsToDelete, List<Type> types, List<Integer> readerColumnIndex) throws IOException {
    int row = rowsToDelete.nextClearBit(0);
    long rowCount = 0;
    long uncompressedSize = 0;
    while (true) {
        if (Thread.currentThread().isInterrupted()) {
            throw new InterruptedIOException();
        }
        int batchSize = reader.nextBatch();
        if (batchSize <= 0) {
            break;
        }
        if (row >= (reader.getFilePosition() + batchSize)) {
            continue;
        }
        Block[] blocks = new Block[types.size()];
        for (int i = 0; i < types.size(); i++) {
            // read from existing columns
            blocks[i] = reader.readBlock(readerColumnIndex.get(i));
        }
        row = toIntExact(reader.getFilePosition());
        Page page = maskedPage(blocks, rowsToDelete, row, batchSize);
        writer.write(page);
        rowCount += page.getPositionCount();
        uncompressedSize += page.getLogicalSizeInBytes();
        row = rowsToDelete.nextClearBit(row + batchSize);
    }
    return new OrcFileInfo(rowCount, uncompressedSize);
}

Example 74 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class OrcFileRewriter method maskedPage.

private static Page maskedPage(Block[] blocks, BitSet rowsToDelete, int start, int count) {
    int[] ids = new int[count];
    int size = 0;
    for (int i = 0; i < count; i++) {
        if (!rowsToDelete.get(start + i)) {
            ids[size] = i;
            size++;
        }
    }
    return new Page(blocks).getPositions(ids, 0, size);
}

Example 75 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class RecordPageSource method getNextPage.

@Override
public Page getNextPage() {
    if (!closed) {
        int i;
        for (i = 0; i < ROWS_PER_REQUEST; i++) {
            if (pageBuilder.isFull()) {
                break;
            }
            if (!cursor.advanceNextPosition()) {
                closed = true;
                break;
            }
            completedPositions++;
            pageBuilder.declarePosition();
            for (int column = 0; column < types.size(); column++) {
                BlockBuilder output = pageBuilder.getBlockBuilder(column);
                if (cursor.isNull(column)) {
                    output.appendNull();
                } else {
                    Type type = types.get(column);
                    Class<?> javaType = type.getJavaType();
                    if (javaType == boolean.class) {
                        type.writeBoolean(output, cursor.getBoolean(column));
                    } else if (javaType == long.class) {
                        type.writeLong(output, cursor.getLong(column));
                    } else if (javaType == double.class) {
                        type.writeDouble(output, cursor.getDouble(column));
                    } else if (javaType == Slice.class) {
                        Slice slice = cursor.getSlice(column);
                        type.writeSlice(output, slice, 0, slice.length());
                    } else {
                        type.writeObject(output, cursor.getObject(column));
                    }
                }
            }
        }
    }
    // only return a page if the buffer is full or we are finishing
    if (pageBuilder.isEmpty() || (!closed && !pageBuilder.isFull())) {
        return null;
    }
    Page page = pageBuilder.build();
    pageBuilder.reset();
    return page;
}