Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 61 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestPinotSegmentPageSource method testMultivaluedType.

@Test
public void testMultivaluedType() throws IOException {
    String[] columnNames = { "col1", "col2" };
    DataSchema.ColumnDataType[] columnDataTypes = { DataSchema.ColumnDataType.INT_ARRAY, DataSchema.ColumnDataType.STRING_ARRAY };
    DataSchema dataSchema = new DataSchema(columnNames, columnDataTypes);
    String[] stringArray = { "stringVal1", "stringVal2" };
    int[] intArray = { 10, 34, 67 };
    DataTableBuilder dataTableBuilder = new DataTableBuilder(dataSchema);
    dataTableBuilder.startRow();
    dataTableBuilder.setColumn(0, intArray);
    dataTableBuilder.setColumn(1, stringArray);
    dataTableBuilder.finishRow();
    DataTable dataTable = dataTableBuilder.build();
    PinotSessionProperties pinotSessionProperties = new PinotSessionProperties(pinotConfig);
    ConnectorSession session = new TestingConnectorSession(pinotSessionProperties.getSessionProperties());
    List<PinotColumnHandle> pinotColumnHandles = ImmutableList.of(new PinotColumnHandle(columnNames[0], PinotColumnUtils.getPrestoTypeFromPinotType(getFieldSpec(columnNames[0], columnDataTypes[0]), false, false), PinotColumnHandle.PinotColumnType.REGULAR), new PinotColumnHandle(columnNames[1], PinotColumnUtils.getPrestoTypeFromPinotType(getFieldSpec(columnNames[1], columnDataTypes[1]), false, false), PinotColumnHandle.PinotColumnType.REGULAR));
    PinotSplit mockPinotSplit = new PinotSplit(pinotConnectorId.toString(), PinotSplit.SplitType.SEGMENT, pinotColumnHandles, Optional.empty(), Optional.of("blah"), ImmutableList.of("seg"), Optional.of("host"), getGrpcPort());
    PinotSegmentPageSource pinotSegmentPageSource = getPinotSegmentPageSource(session, ImmutableList.of(dataTable), mockPinotSplit, pinotColumnHandles);
    Page page = requireNonNull(pinotSegmentPageSource.getNextPage(), "Expected a valid page");
    for (int i = 0; i < columnDataTypes.length; i++) {
        Block block = page.getBlock(i);
        Type type = PinotColumnUtils.getPrestoTypeFromPinotType(getFieldSpec(columnNames[i], columnDataTypes[i]), false, false);
        Assert.assertTrue(type instanceof ArrayType, "presto type should be array");
        if (((ArrayType) type).getElementType() instanceof IntegerType) {
            Assert.assertTrue(block.getBlock(0).getInt(0) == 10, "Array element not matching");
            Assert.assertTrue(block.getBlock(0).getInt(1) == 34, "Array element not matching");
            Assert.assertTrue(block.getBlock(0).getInt(2) == 67, "Array element not matching");
        } else if (((ArrayType) type).getElementType() instanceof VariableWidthType) {
            Type type1 = ((ArrayType) type).getElementType();
            Assert.assertTrue(block.getBlock(0) instanceof VariableWidthBlock);
            VariableWidthBlock variableWidthBlock = (VariableWidthBlock) block.getBlock(0);
            Assert.assertTrue("stringVal1".equals(new String(variableWidthBlock.getSlice(0, 0, variableWidthBlock.getSliceLength(0)).getBytes())), "Array element not matching");
            Assert.assertTrue("stringVal2".equals(new String(variableWidthBlock.getSlice(1, 0, variableWidthBlock.getSliceLength(1)).getBytes())), "Array element not matching");
        }
    }
}

Example 62 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestPinotSegmentPageSource method testAllDataTypes.

@Test
public void testAllDataTypes() {
    PinotSessionProperties pinotSessionProperties = new PinotSessionProperties(pinotConfig);
    ConnectorSession session = new TestingConnectorSession(pinotSessionProperties.getSessionProperties());
    List<DataTable> dataTables = IntStream.range(0, 3).mapToObj(i -> createDataTableWithAllTypes()).collect(toImmutableList());
    List<PinotColumnHandle> pinotColumnHandles = createPinotColumnHandlesWithAllTypes();
    PinotSplit mockPinotSplit = new PinotSplit(pinotConnectorId.toString(), PinotSplit.SplitType.SEGMENT, pinotColumnHandles, Optional.empty(), Optional.of("blah"), ImmutableList.of("seg"), Optional.of("host"), getGrpcPort());
    PinotSegmentPageSource pinotSegmentPageSource = getPinotSegmentPageSource(session, dataTables, mockPinotSplit, pinotColumnHandles);
    for (int i = 0; i < dataTables.size(); ++i) {
        Page page = requireNonNull(pinotSegmentPageSource.getNextPage(), "Expected a valid page");
        for (int j = 0; j < ALL_TYPES.size(); ++j) {
            Block block = page.getBlock(j);
            Type type = PinotColumnUtils.getPrestoTypeFromPinotType(getFieldSpec("dontcare", ALL_TYPES.get(j)), false, false);
            long maxHashCode = Long.MIN_VALUE;
            for (int k = 0; k < NUM_ROWS; ++k) {
                maxHashCode = Math.max(type.hash(block, k), maxHashCode);
            }
            Assert.assertTrue(maxHashCode != 0, "Not all column values can have hash code 0");
        }
    }
}

Example 63 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class OrcFileWriter method appendPages.

@Override
public void appendPages(List<Page> inputPages, int[] pageIndexes, int[] positionIndexes) {
    checkArgument(pageIndexes.length == positionIndexes.length, "pageIndexes and positionIndexes do not match");
    for (int i = 0; i < pageIndexes.length; i++) {
        Page page = inputPages.get(pageIndexes[i]);
        // This will do data copy; be aware
        Page singleValuePage = page.getSingleValuePage(positionIndexes[i]);
        try {
            orcWriter.write(singleValuePage);
            uncompressedSize += singleValuePage.getLogicalSizeInBytes();
            rowCount++;
        } catch (IOException | UncheckedIOException e) {
            throw new PrestoException(RAPTOR_WRITER_DATA_ERROR, e);
        }
    }
}

Example 64 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class ShardCompactor method compact.

private List<ShardInfo> compact(StoragePageSink storagePageSink, boolean tableSupportsDeltaDelete, OptionalInt bucketNumber, Map<UUID, Optional<UUID>> uuidsMap, List<Long> columnIds, List<Type> columnTypes) throws IOException {
    for (Map.Entry<UUID, Optional<UUID>> entry : uuidsMap.entrySet()) {
        UUID uuid = entry.getKey();
        Optional<UUID> deltaUuid = entry.getValue();
        try (ConnectorPageSource pageSource = storageManager.getPageSource(DEFAULT_RAPTOR_CONTEXT, DEFAULT_HIVE_FILE_CONTEXT, uuid, deltaUuid, tableSupportsDeltaDelete, bucketNumber, columnIds, columnTypes, TupleDomain.all(), readerAttributes)) {
            while (!pageSource.isFinished()) {
                Page page = pageSource.getNextPage();
                if (isNullOrEmptyPage(page)) {
                    continue;
                }
                storagePageSink.appendPages(ImmutableList.of(page));
                if (storagePageSink.isFull()) {
                    storagePageSink.flush();
                }
            }
        }
    }
    return getFutureValue(storagePageSink.commit());
}

Example 65 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class OrcSelectiveRecordReader method evaluateDeterministicFilterFunctionWithConstantInputs.

private boolean evaluateDeterministicFilterFunctionWithConstantInputs(FilterFunction function) {
    int[] inputs = function.getInputChannels();
    Block[] blocks = new Block[inputs.length];
    for (int i = 0; i < inputs.length; i++) {
        int columnIndex = filterFunctionInputMapping.get(inputs[i]);
        Object constantValue = constantValues[columnIndex];
        blocks[i] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValue == NULL_MARKER ? null : constantValue, 1);
    }
    initializeTmpErrors(1);
    int positionCount = function.filter(new Page(blocks), new int[] { 0 }, 1, tmpErrors);
    if (tmpErrors[0] != null) {
        constantFilterError = tmpErrors[0];
    }
    return positionCount == 1;
}