Examples with PageBuilder io.prestosql.spi.PageBuilder used on opensource projects

Example 6 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class TableFinishOperator method getOutput.

@Override
public Page getOutput() {
    if (snapshotState != null) {
        Page marker = snapshotState.nextMarker();
        if (marker != null) {
            return marker;
        }
    }
    if (!isBlocked().isDone()) {
        return null;
    }
    if (!statisticsAggregationOperator.isFinished()) {
        verify(statisticsAggregationOperator.isBlocked().isDone(), "aggregation operator should not be blocked");
        OperationTimer timer = new OperationTimer(statisticsCpuTimerEnabled);
        Page page = statisticsAggregationOperator.getOutput();
        timer.end(statisticsTiming);
        if (page == null) {
            return null;
        }
        for (int position = 0; position < page.getPositionCount(); position++) {
            computedStatistics.add(getComputedStatistics(page, position));
        }
        return null;
    }
    if (state != State.FINISHING) {
        return null;
    }
    state = State.FINISHED;
    outputMetadata = tableFinisher.finishTable(ImmutableList.copyOf(fragment), ImmutableList.copyOf(computedStatistics));
    // output page will only be constructed once,
    // so a new PageBuilder is constructed (instead of using PageBuilder.reset)
    PageBuilder page = new PageBuilder(1, TYPES);
    page.declarePosition();
    BIGINT.writeLong(page.getBlockBuilder(0), rowCount);
    return page.build();
}

Example 7 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class RowNumberOperator method getSelectedRows.

private Page getSelectedRows() {
    verify(selectedRowPageBuilder.isPresent());
    int rowNumberChannel = types.size() - 1;
    PageBuilder pageBuilder = selectedRowPageBuilder.get();
    verify(pageBuilder.isEmpty());
    for (int currentPosition = 0; currentPosition < inputPage.getPositionCount(); currentPosition++) {
        long partitionId = getPartitionId(currentPosition);
        long rowCount = partitionRowCount.get(partitionId);
        if (rowCount == maxRowsPerPartition.get()) {
            continue;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < outputChannels.length; i++) {
            int channel = outputChannels[i];
            Type type = types.get(i);
            type.appendTo(inputPage.getBlock(channel), currentPosition, pageBuilder.getBlockBuilder(i));
        }
        BIGINT.writeLong(pageBuilder.getBlockBuilder(rowNumberChannel), rowCount + 1);
        partitionRowCount.set(partitionId, rowCount + 1);
    }
    if (pageBuilder.isEmpty()) {
        return null;
    }
    Page page = pageBuilder.build();
    pageBuilder.reset();
    return page;
}

Example 8 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class MapConcatFunction method mapConcat.

@UsedByGeneratedCode
public static Block mapConcat(MapType mapType, Object state, Block[] maps) {
    int entries = 0;
    int lastMapIndex = maps.length - 1;
    int firstMapIndex = lastMapIndex;
    for (int i = 0; i < maps.length; i++) {
        entries += maps[i].getPositionCount();
        if (maps[i].getPositionCount() > 0) {
            lastMapIndex = i;
            firstMapIndex = min(firstMapIndex, i);
        }
    }
    if (lastMapIndex == firstMapIndex) {
        return maps[lastMapIndex];
    }
    PageBuilder pageBuilder = (PageBuilder) state;
    if (pageBuilder.isFull()) {
        pageBuilder.reset();
    }
    // TODO: we should move TypedSet into user state as well
    Type keyType = mapType.getKeyType();
    Type valueType = mapType.getValueType();
    TypedSet typedSet = new TypedSet(keyType, entries / 2, FUNCTION_NAME);
    BlockBuilder mapBlockBuilder = pageBuilder.getBlockBuilder(0);
    BlockBuilder blockBuilder = mapBlockBuilder.beginBlockEntry();
    // the last map
    Block map = maps[lastMapIndex];
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        typedSet.add(map, i);
        keyType.appendTo(map, i, blockBuilder);
        valueType.appendTo(map, i + 1, blockBuilder);
    }
    // the map between the last and the first
    for (int idx = lastMapIndex - 1; idx > firstMapIndex; idx--) {
        map = maps[idx];
        for (int i = 0; i < map.getPositionCount(); i += 2) {
            if (!typedSet.contains(map, i)) {
                typedSet.add(map, i);
                keyType.appendTo(map, i, blockBuilder);
                valueType.appendTo(map, i + 1, blockBuilder);
            }
        }
    }
    // the first map
    map = maps[firstMapIndex];
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        if (!typedSet.contains(map, i)) {
            keyType.appendTo(map, i, blockBuilder);
            valueType.appendTo(map, i + 1, blockBuilder);
        }
    }
    mapBlockBuilder.closeEntry();
    pageBuilder.declarePosition();
    return mapType.getObject(mapBlockBuilder, mapBlockBuilder.getPositionCount() - 1);
}

Example 9 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveConfig config, HiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(HiveTestUtils.TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(round(stats.getInputPageSizeInBytes().getAllTime().getMax()), page.getRetainedSizeInBytes());
    return length;
}

Example 10 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class Query method getNextResult.

private synchronized DataCenterQueryResults getNextResult(long token, DataSize targetResultSize) {
    // check if the result for the token have already been created
    Optional<DataCenterQueryResults> cachedResult = getCachedDCQueryResult(token);
    if (cachedResult.isPresent()) {
        return cachedResult.get();
    }
    verify(nextToken.isPresent(), "Can not generate next result when next token is not present");
    verify(token == nextToken.getAsLong(), "Expected token to equal next token");
    URI queryHtmlUri = null;
    try {
        queryHtmlUri = new URI("http://localhost");
    } catch (URISyntaxException e) {
        log.error("get uri error: %s", e.getMessage());
    }
    // Remove as many pages as possible from the exchange until just greater than DESIRED_RESULT_BYTES
    // NOTE: it is critical that query results are created for the pages removed from the exchange
    // client while holding the lock because the query may transition to the finished state when the
    // last page is removed.  If another thread observes this state before the response is cached
    // the pages will be lost.
    List<SerializedPage> data = null;
    try {
        ImmutableList.Builder<SerializedPage> builder = new ImmutableList.Builder<>();
        long bytes = 0;
        long rows = 0;
        long targetResultBytes = targetResultSize.toBytes();
        while (bytes < targetResultBytes) {
            // at this point, origin is irrelevant, so we can safely ignore it
            SerializedPage serializedPage = exchangeClient.pollPage(null).getLeft();
            if (serializedPage == null) {
                break;
            }
            builder.add(new SerializedPage(serializedPage.getSlice().getBytes(), serializedPage.getPageCodecMarkers(), serializedPage.getPositionCount(), serializedPage.getUncompressedSizeInBytes()));
            bytes += serializedPage.getUncompressedSizeInBytes();
            rows += serializedPage.getPositionCount();
        }
        if (rows > 0) {
            // client implementations do not properly handle empty list of data
            data = builder.build();
        }
    } catch (Throwable cause) {
        queryManager.failQuery(queryId, cause);
    }
    // get the query info before returning
    // force update if query manager is closed
    QueryInfo queryInfo = queryManager.getFullQueryInfo(queryId);
    queryManager.recordHeartbeat(queryId);
    closeExchangeClientIfNecessary(queryInfo);
    // for queries with no output, return a fake result for clients that require it
    if ((queryInfo.getState() == QueryState.FINISHED) && !queryInfo.getOutputStage().isPresent()) {
        columns = ImmutableList.of(createColumn("result", BooleanType.BOOLEAN));
        PageBuilder pageBuilder = new PageBuilder(Collections.singletonList(BooleanType.BOOLEAN));
        pageBuilder.declarePosition();
        BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0);
        BooleanType.BOOLEAN.writeBoolean(blockBuilder, true);
        Page page = pageBuilder.build();
        SerializedPage serializedPage = serde.serialize(page);
        data = Collections.singletonList(new SerializedPage(serializedPage.getSlice().getBytes(), serializedPage.getPageCodecMarkers(), serializedPage.getPositionCount(), serializedPage.getUncompressedSizeInBytes()));
    } else if (queryInfo.isRunningAsync()) {
        columns = ImmutableList.of(createColumn("result", BooleanType.BOOLEAN), createColumn("runningAsync", BooleanType.BOOLEAN));
        PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(BooleanType.BOOLEAN, BooleanType.BOOLEAN));
        pageBuilder.declarePosition();
        BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0);
        BooleanType.BOOLEAN.writeBoolean(blockBuilder, true);
        BooleanType.BOOLEAN.writeBoolean(blockBuilder, true);
        Page page = pageBuilder.build();
        SerializedPage serializedPage = serde.serialize(page);
        data = Collections.singletonList(new SerializedPage(serializedPage.getSlice().getBytes(), serializedPage.getPageCodecMarkers(), serializedPage.getPositionCount(), serializedPage.getUncompressedSizeInBytes()));
    }
    // (3) Query supports and started running Async
    if ((!queryInfo.isFinalQueryInfo() && queryInfo.getState() != FAILED && !queryInfo.isRunningAsync()) || !exchangeClient.isClosed()) {
        nextToken = OptionalLong.of(token + 1);
    } else {
        nextToken = OptionalLong.empty();
    }
    URI nextResultsUri = null;
    if (nextToken.isPresent()) {
        try {
            nextResultsUri = new URI(Long.toString(nextToken.getAsLong()));
        } catch (URISyntaxException e) {
            log.error("get uri error: %s", e.getMessage());
        }
    }
    // update catalog, schema, and path
    setCatalog = queryInfo.getSetCatalog();
    setSchema = queryInfo.getSetSchema();
    setPath = queryInfo.getSetPath();
    // update setSessionProperties
    setSessionProperties = queryInfo.getSetSessionProperties();
    resetSessionProperties = queryInfo.getResetSessionProperties();
    // update setRoles
    setRoles = queryInfo.getSetRoles();
    // update preparedStatements
    addedPreparedStatements = queryInfo.getAddedPreparedStatements();
    deallocatedPreparedStatements = queryInfo.getDeallocatedPreparedStatements();
    // update startedTransactionId
    startedTransactionId = queryInfo.getStartedTransactionId();
    clearTransactionId = queryInfo.isClearTransactionId();
    // first time through, self is null
    DataCenterQueryResults queryResults = new DataCenterQueryResults(queryId.toString(), queryHtmlUri, findCancelableLeafStage(queryInfo), nextResultsUri, columns, data, toStatementStats(queryInfo), toQueryError(queryInfo), mappedCopy(queryInfo.getWarnings(), Query::toClientWarning), queryInfo.getUpdateType(), true);
    // cache the new result
    lastToken = token;
    lastDCQueryResult = queryResults;
    return queryResults;
}