Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 16 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestOrcWriter method testVerifyNoIllegalStateException.

@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = "Dummy exception from mocked instance")
public void testVerifyNoIllegalStateException() throws IOException {
    OrcWriter writer = new OrcWriter(new MockDataSink(), ImmutableList.of("test1"), ImmutableList.of(VARCHAR), ORC, NONE, Optional.empty(), NO_ENCRYPTION, OrcWriterOptions.builder().withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMinSize(new DataSize(0, MEGABYTE)).withStripeMaxSize(new DataSize(32, MEGABYTE)).withStripeMaxRowCount(10).build()).withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE).withDictionaryMaxMemory(new DataSize(32, MEGABYTE)).build(), ImmutableMap.of(), HIVE_STORAGE_TIME_ZONE, false, null, new OrcWriterStats());
    int entries = 65536;
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, entries);
    byte[] bytes = "dummyString".getBytes();
    for (int j = 0; j < entries; j++) {
        // force to write different data
        bytes[0] = (byte) ((bytes[0] + 1) % 128);
        blockBuilder.writeBytes(Slices.wrappedBuffer(bytes, 0, bytes.length), 0, bytes.length);
        blockBuilder.closeEntry();
    }
    Block[] blocks = new Block[] { blockBuilder.build() };
    try {
        // Throw IOException after first flush
        writer.write(new Page(blocks));
    } catch (IOException e) {
        writer.close();
    }
}

Example 17 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class PrestoSparkShufflePageInput method createPage.

private static Page createPage(int rowCount, BasicSliceInput input, List<Type> types) {
    checkArgument(rowCount > 0, "rowCount must be greater than zero: %s", rowCount);
    if (input.length() == 0) {
        // zero column page
        verify(types.isEmpty(), "types is expected to be empty");
        return new Page(rowCount);
    }
    PageBuilder pageBuilder = new PageBuilder(types);
    while (input.isReadable()) {
        pageBuilder.declarePosition();
        for (int channel = 0; channel < types.size(); channel++) {
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(channel);
            blockBuilder.readPositionFrom(input);
        }
    }
    Page page = pageBuilder.build();
    verify(page.getPositionCount() == rowCount, "unexpected row count: %s != %s", page.getPositionCount(), rowCount);
    return page;
}

Example 18 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class PrestoSparkDiskPageInput method loadBroadcastTable.

private List<Page> loadBroadcastTable(List<PrestoSparkStorageHandle> broadcastTaskFilesInfo, TempStorage tempStorage, TempDataOperationContext tempDataOperationContext, UpdateMemory updateMemory) {
    try {
        CRC32 checksum = new CRC32();
        ImmutableList.Builder<Page> pages = ImmutableList.builder();
        List<PrestoSparkStorageHandle> broadcastTaskFilesInfoCopy = new ArrayList<>(broadcastTaskFilesInfo);
        shuffle(broadcastTaskFilesInfoCopy);
        for (PrestoSparkTaskOutput taskFileInfo : broadcastTaskFilesInfoCopy) {
            checksum.reset();
            PrestoSparkStorageHandle prestoSparkStorageHandle = (PrestoSparkStorageHandle) taskFileInfo;
            TempStorageHandle tempStorageHandle = tempStorage.deserialize(prestoSparkStorageHandle.getSerializedStorageHandle());
            log.info("Reading path: " + tempStorageHandle.toString());
            try (InputStream inputStream = tempStorage.open(tempDataOperationContext, tempStorageHandle);
                InputStreamSliceInput inputStreamSliceInput = new InputStreamSliceInput(inputStream)) {
                Iterator<SerializedPage> pagesIterator = readSerializedPages(inputStreamSliceInput);
                while (pagesIterator.hasNext()) {
                    SerializedPage serializedPage = pagesIterator.next();
                    checksum.update(serializedPage.getSlice().byteArray(), serializedPage.getSlice().byteArrayOffset(), serializedPage.getSlice().length());
                    Page deserializedPage = pagesSerde.deserialize(serializedPage);
                    pages.add(deserializedPage);
                    stagingBroadcastTableSizeInBytes += deserializedPage.getRetainedSizeInBytes();
                }
                updateMemory.update();
            }
            if (checksum.getValue() != prestoSparkStorageHandle.getChecksum()) {
                throw new PrestoException(STORAGE_ERROR, "Disk page checksum does not match. " + "Data seems to be corrupted on disk for file " + tempStorageHandle.toString());
            }
        }
        return pages.build();
    } catch (UncheckedIOException | IOException e) {
        throw new PrestoException(STORAGE_ERROR, "Unable to read data from disk: ", e);
    }
}

Example 19 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class PrestoSparkRowOutputOperator method addInput.

@Override
public void addInput(Page page) {
    page = pagePreprocessor.apply(page);
    int positionCount = page.getPositionCount();
    if (positionCount == 0) {
        return;
    }
    int partitionCount = partitionFunction.getPartitionCount();
    if (rowBatchBuilder == null) {
        rowBatchBuilder = PrestoSparkRowBatch.builder(partitionCount, targetAverageRowSizeInBytes);
    }
    int channelCount = page.getChannelCount();
    Page partitionFunctionArguments = getPartitionFunctionArguments(page);
    for (int position = 0; position < positionCount; position++) {
        if (rowBatchBuilder.isFull()) {
            outputBuffer.enqueue(rowBatchBuilder.build());
            rowBatchBuilder = PrestoSparkRowBatch.builder(partitionCount, targetAverageRowSizeInBytes);
        }
        SliceOutput output = rowBatchBuilder.beginRowEntry();
        for (int channel = 0; channel < channelCount; channel++) {
            Block block = page.getBlock(channel);
            block.writePositionTo(position, output);
        }
        boolean shouldReplicate = (replicateNullsAndAny && !hasAnyRowBeenReplicated) || nullChannel.isPresent() && page.getBlock(nullChannel.getAsInt()).isNull(position);
        if (shouldReplicate) {
            hasAnyRowBeenReplicated = true;
            rowBatchBuilder.closeEntryForReplicatedRow();
        } else {
            int partition = getPartition(partitionFunctionArguments, position);
            rowBatchBuilder.closeEntryForNonReplicatedRow(partition);
        }
    }
    updateMemoryContext();
}

Example 20 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class DruidBrokerPageSource method getNextPage.

@Override
public Page getNextPage() {
    if (finished) {
        return null;
    }
    long start = System.nanoTime();
    boolean columnHandlesHasErrorMessageField = columnHandles.stream().anyMatch(handle -> ((DruidColumnHandle) handle).getColumnName().equals("errorMessage"));
    try {
        String readLine;
        while ((readLine = responseStream.readLine()) != null) {
            // if read a blank line,it means read finish
            if (readLine.isEmpty()) {
                finished = true;
                break;
            } else {
                JsonNode rootNode = OBJECT_MAPPER.readTree(readLine);
                if (rootNode.has("errorMessage") && !columnHandlesHasErrorMessageField) {
                    throw new PrestoException(DRUID_BROKER_RESULT_ERROR, rootNode.findValue("errorMessage").asText());
                }
                for (int i = 0; i < columnHandles.size(); i++) {
                    Type type = columnTypes.get(i);
                    BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
                    JsonNode value = rootNode.get(((DruidColumnHandle) columnHandles.get(i)).getColumnName());
                    if (value == null) {
                        blockBuilder.appendNull();
                        continue;
                    }
                    if (type instanceof BigintType) {
                        type.writeLong(blockBuilder, value.longValue());
                    } else if (type instanceof DoubleType) {
                        type.writeDouble(blockBuilder, value.doubleValue());
                    } else if (type instanceof RealType) {
                        type.writeLong(blockBuilder, floatToRawIntBits(value.floatValue()));
                    } else if (type instanceof TimestampType) {
                        DateTimeFormatter formatter = ISODateTimeFormat.dateTimeParser().withChronology(ISOChronology.getInstanceUTC()).withOffsetParsed();
                        DateTime dateTime = formatter.parseDateTime(value.textValue());
                        type.writeLong(blockBuilder, dateTime.getMillis());
                    } else {
                        Slice slice = Slices.utf8Slice(value.textValue());
                        type.writeSlice(blockBuilder, slice);
                    }
                }
            }
            pageBuilder.declarePosition();
            if (pageBuilder.isFull()) {
                break;
            }
        }
        // if responseStream.readLine() is null, it means read finish
        if (readLine == null) {
            finished = true;
            return null;
        }
        // only return a page if the buffer is full or we are finishing
        if (pageBuilder.isEmpty() || (!finished && !pageBuilder.isFull())) {
            return null;
        }
        Page page = pageBuilder.build();
        completedPositions += page.getPositionCount();
        completedBytes += page.getSizeInBytes();
        pageBuilder.reset();
        return page;
    } catch (IOException e) {
        finished = true;
        throw new PrestoException(DRUID_BROKER_RESULT_ERROR, "Parse druid client response error", e);
    } finally {
        readTimeNanos += System.nanoTime() - start;
    }
}