Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 81 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class AbstractTestHiveClient method doTestBucketSortedTables.

private void doTestBucketSortedTables(SchemaTableName table, boolean useTempPath, HiveStorageFormat storageFormat) throws IOException {
    int bucketCount = 3;
    int expectedRowCount = 0;
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession(ImmutableMap.of(SORTED_WRITE_TO_TEMP_PATH_ENABLED, useTempPath));
        // begin creating the table
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(table, ImmutableList.<ColumnMetadata>builder().add(new ColumnMetadata("id", VARCHAR)).add(new ColumnMetadata("value_asc", VARCHAR)).add(new ColumnMetadata("value_desc", BIGINT)).add(new ColumnMetadata("ds", VARCHAR)).build(), ImmutableMap.<String, Object>builder().put(STORAGE_FORMAT_PROPERTY, storageFormat).put(PARTITIONED_BY_PROPERTY, ImmutableList.of("ds")).put(BUCKETED_BY_PROPERTY, ImmutableList.of("id")).put(BUCKET_COUNT_PROPERTY, bucketCount).put(SORTED_BY_PROPERTY, ImmutableList.builder().add(new SortingColumn("value_asc", SortingColumn.Order.ASCENDING)).add(new SortingColumn("value_desc", SortingColumn.Order.DESCENDING)).build()).build());
        HiveOutputTableHandle outputHandle = (HiveOutputTableHandle) metadata.beginCreateTable(session, tableMetadata, Optional.empty());
        // write the data
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle, TEST_HIVE_PAGE_SINK_CONTEXT);
        List<Type> types = tableMetadata.getColumns().stream().map(ColumnMetadata::getType).collect(toList());
        ThreadLocalRandom random = ThreadLocalRandom.current();
        for (int i = 0; i < 50; i++) {
            MaterializedResult.Builder builder = MaterializedResult.resultBuilder(session, types);
            for (int j = 0; j < 1000; j++) {
                builder.row(sha256().hashLong(random.nextLong()).toString(), "test" + random.nextInt(100), random.nextLong(100_000), "2018-04-01");
                expectedRowCount++;
            }
            sink.appendPage(builder.build().toPage());
        }
        // verify we have enough temporary files per bucket to require multiple passes
        Path path = useTempPath ? getTempFilePathRoot(outputHandle).get() : getStagingPathRoot(outputHandle);
        HdfsContext context = new HdfsContext(session, table.getSchemaName(), table.getTableName(), outputHandle.getLocationHandle().getTargetPath().toString(), true);
        Set<String> files = listAllDataFiles(context, path);
        assertThat(listAllDataFiles(context, path)).filteredOn(file -> file.contains(".tmp-sort")).size().isGreaterThan(bucketCount * getHiveClientConfig().getMaxOpenSortFiles() * 2);
        // finish the write
        Collection<Slice> fragments = getFutureValue(sink.finish());
        // verify there are no temporary files
        for (String file : listAllDataFiles(context, path)) {
            assertThat(file).doesNotContain(".tmp-sort.");
        }
        // finish creating table
        metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of());
        transaction.commit();
    }
    // verify that bucket files are sorted
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession(ImmutableMap.of(SORTED_WRITE_TO_TEMP_PATH_ENABLED, useTempPath));
        ConnectorTableHandle hiveTableHandle = getTableHandle(metadata, table);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, hiveTableHandle).values());
        ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, hiveTableHandle, TupleDomain.all());
        List<ConnectorSplit> splits = getAllSplits(session, transaction, layoutHandle);
        assertThat(splits).hasSize(bucketCount);
        int actualRowCount = 0;
        for (ConnectorSplit split : splits) {
            try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, layoutHandle, columnHandles, NON_CACHEABLE)) {
                String lastValueAsc = null;
                long lastValueDesc = -1;
                while (!pageSource.isFinished()) {
                    Page page = pageSource.getNextPage();
                    if (page == null) {
                        continue;
                    }
                    for (int i = 0; i < page.getPositionCount(); i++) {
                        Block blockAsc = page.getBlock(1);
                        Block blockDesc = page.getBlock(2);
                        assertFalse(blockAsc.isNull(i));
                        assertFalse(blockDesc.isNull(i));
                        String valueAsc = VARCHAR.getSlice(blockAsc, i).toStringUtf8();
                        if (lastValueAsc != null) {
                            assertGreaterThanOrEqual(valueAsc, lastValueAsc);
                            if (valueAsc.equals(lastValueAsc)) {
                                long valueDesc = BIGINT.getLong(blockDesc, i);
                                if (lastValueDesc != -1) {
                                    assertLessThanOrEqual(valueDesc, lastValueDesc);
                                }
                                lastValueDesc = valueDesc;
                            } else {
                                lastValueDesc = -1;
                            }
                        }
                        lastValueAsc = valueAsc;
                        actualRowCount++;
                    }
                }
            }
        }
        assertThat(actualRowCount).isEqualTo(expectedRowCount);
    }
}

Example 82 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class AccumulatorCompiler method generateAddInput.

private static void generateAddInput(ClassDefinition definition, List<FieldDefinition> stateField, List<FieldDefinition> inputChannelFields, FieldDefinition maskChannelField, List<ParameterMetadata> parameterMetadatas, List<Class> lambdaInterfaces, List<FieldDefinition> lambdaProviderFields, MethodHandle inputFunction, CallSiteBinder callSiteBinder, boolean grouped) {
    ImmutableList.Builder<Parameter> parameters = ImmutableList.builder();
    if (grouped) {
        parameters.add(arg("groupIdsBlock", GroupByIdBlock.class));
    }
    Parameter page = arg("page", Page.class);
    parameters.add(page);
    MethodDefinition method = definition.declareMethod(a(PUBLIC), "addInput", type(void.class), parameters.build());
    Scope scope = method.getScope();
    BytecodeBlock body = method.getBody();
    Variable thisVariable = method.getThis();
    if (grouped) {
        generateEnsureCapacity(scope, stateField, body);
    }
    Variable masksBlock = scope.declareVariable(Block.class, "masksBlock");
    body.comment("masksBlock = extractMaskBlock(%s, page);", maskChannelField.getName()).append(thisVariable.getField(maskChannelField)).append(page).invokeStatic(AggregationUtils.class, "extractMaskBlock", Block.class, int.class, Page.class).putVariable(masksBlock);
    int inputChannelsSize = inputChannelFields.size();
    ImmutableList.Builder<Variable> variablesBuilder = ImmutableList.builderWithExpectedSize(inputChannelsSize);
    for (int i = 0; i < inputChannelFields.size(); i++) {
        FieldDefinition inputChannelField = inputChannelFields.get(i);
        Variable blockVariable = scope.declareVariable(Block.class, "block" + i);
        variablesBuilder.add(blockVariable);
        body.comment("%s = page.getBlock(%s);", blockVariable.getName(), inputChannelField.getName()).append(page).append(thisVariable.getField(inputChannelField)).invokeVirtual(Page.class, "getBlock", Block.class, int.class).putVariable(blockVariable);
    }
    List<Variable> parameterVariables = variablesBuilder.build();
    BytecodeBlock block = generateInputForLoop(stateField, parameterMetadatas, inputFunction, scope, parameterVariables, lambdaInterfaces, lambdaProviderFields, masksBlock, callSiteBinder, grouped);
    body.append(block);
    body.ret();
}

Example 83 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TableWriterMergeOperator method getOutput.

@Override
public Page getOutput() {
    // pass through fragment pages first to avoid use extra memory
    if (!fragmentsBlocks.isEmpty()) {
        Block fragmentsBlock = fragmentsBlocks.poll();
        systemMemoryContext.setBytes(getRetainedMemoryBytes());
        return createFragmentsPage(fragmentsBlock);
    }
    // still working on merging statistic pages
    if (!isBlocked().isDone()) {
        return null;
    }
    if (!statisticsAggregationOperator.isFinished()) {
        verify(statisticsAggregationOperator.isBlocked().isDone(), "aggregation operator should not be blocked");
        OperationTimer timer = new OperationTimer(statisticsCpuTimerEnabled);
        Page page = statisticsAggregationOperator.getOutput();
        timer.end(statisticsTiming);
        if (page == null) {
            return null;
        }
        return createStatisticsPage(types, page, createTableCommitContext(false));
    }
    if (state != State.FINISHING) {
        return null;
    }
    state = State.FINISHED;
    Page finalPage = createFinalPage();
    systemMemoryContext.setBytes(getRetainedMemoryBytes());
    return finalPage;
}

Example 84 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TableWriterUtils method extractStatisticsRows.

public static Optional<Page> extractStatisticsRows(Page page) {
    int statisticsPositionCount = 0;
    for (int position = 0; position < page.getPositionCount(); position++) {
        if (isStatisticsPosition(page, position)) {
            statisticsPositionCount++;
        }
    }
    if (statisticsPositionCount == 0) {
        return Optional.empty();
    }
    if (statisticsPositionCount == page.getPositionCount()) {
        return Optional.of(page);
    }
    int selectedPositionsIndex = 0;
    int[] selectedPositions = new int[statisticsPositionCount];
    for (int position = 0; position < page.getPositionCount(); position++) {
        if (isStatisticsPosition(page, position)) {
            selectedPositions[selectedPositionsIndex] = position;
            selectedPositionsIndex++;
        }
    }
    Block[] blocks = new Block[page.getChannelCount()];
    for (int channel = 0; channel < page.getChannelCount(); channel++) {
        blocks[channel] = page.getBlock(channel).getPositions(selectedPositions, 0, statisticsPositionCount);
    }
    return Optional.of(new Page(statisticsPositionCount, blocks));
}

Example 85 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TopNOperator method getOutput.

@Override
public Page getOutput() {
    if (!finishing || noMoreOutput()) {
        return null;
    }
    if (outputIterator == null) {
        // start flushing
        outputIterator = topNBuilder.buildResult();
    }
    Page output = null;
    if (outputIterator.hasNext()) {
        output = outputIterator.next();
    } else {
        outputIterator = emptyIterator();
    }
    updateMemoryReservation();
    return output;
}