Examples with OutputBuffer io.prestosql.execution.buffer.OutputBuffer used on opensource projects

Example 1 with OutputBuffer

use of io.prestosql.execution.buffer.OutputBuffer in project hetu-core by openlookeng.

the class TestTaskOutputOperatorFactory method testDuplicate.

@Test
public void testDuplicate() {
    OutputBuffer outputBuffer = mock(OutputBuffer.class);
    TaskOutputOperatorFactory factory1 = new TaskOutputOperatorFactory(1, new PlanNodeId("planNodeId"), outputBuffer, a -> a);
    OperatorFactory factory2 = factory1.duplicate();
    OperatorFactory factory3 = factory1.duplicate();
    OperatorFactory factory4 = factory2.duplicate();
    factory1.noMoreOperators();
    factory3.noMoreOperators();
    factory4.noMoreOperators();
    verify(outputBuffer, never()).setNoMoreInputChannels();
    factory2.noMoreOperators();
    verify(outputBuffer).setNoMoreInputChannels();
}

Example 2 with OutputBuffer

use of io.prestosql.execution.buffer.OutputBuffer in project hetu-core by openlookeng.

the class TestPartitionedOutputOperatorFactory method testDuplicate.

@Test
public void testDuplicate() {
    OutputBuffer outputBuffer = mock(OutputBuffer.class);
    PartitionedOutputOperatorFactory factory1 = new PartitionedOutputOperatorFactory(1, new PlanNodeId("planNodeId"), Collections.emptyList(), a -> a, mock(PartitionFunction.class), Collections.emptyList(), Collections.emptyList(), false, OptionalInt.empty(), outputBuffer, DataSize.succinctBytes(1));
    OperatorFactory factory2 = factory1.duplicate();
    OperatorFactory factory3 = factory1.duplicate();
    OperatorFactory factory4 = factory2.duplicate();
    factory1.noMoreOperators();
    factory3.noMoreOperators();
    factory4.noMoreOperators();
    verify(outputBuffer, never()).setNoMoreInputChannels();
    factory2.noMoreOperators();
    verify(outputBuffer).setNoMoreInputChannels();
}

Example 3 with OutputBuffer

use of io.prestosql.execution.buffer.OutputBuffer in project hetu-core by openlookeng.

the class TaskOutputOperator method addInput.

@Override
public void addInput(Page page) {
    requireNonNull(page, "page is null");
    Page inputPage = page;
    if (snapshotState != null) {
        if (snapshotState.processPage(inputPage)) {
            inputPage = snapshotState.nextMarker();
        }
    }
    if (inputPage.getPositionCount() == 0) {
        return;
    }
    if (!(inputPage instanceof MarkerPage)) {
        inputPage = pagePreprocessor.apply(inputPage);
    } else if (isStage0) {
        // Do not add marker to final output.
        return;
    }
    List<SerializedPage> serializedPages = splitPage(inputPage, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream().map(p -> serde.serialize(p)).collect(toImmutableList());
    if (inputPage instanceof MarkerPage) {
        // The result is that for buffer #2, it receives marker 2 before marker 1.
        synchronized (outputBuffer) {
            outputBuffer.enqueue(serializedPages, id);
        }
    } else {
        outputBuffer.enqueue(serializedPages, id);
    }
    operatorContext.recordOutput(inputPage.getSizeInBytes(), inputPage.getPositionCount());
}

Example 4 with OutputBuffer

use of io.prestosql.execution.buffer.OutputBuffer in project boostkit-bigdata by kunpengcompute.

the class OmniLocalQueryRunner method createDriversWithPlanOnly.

private void createDriversWithPlanOnly(Session session, Plan plan, OutputFactory outputFactory, TaskContext taskContext) {
    if (printPlan) {
        System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata, plan.getStatsAndCosts(), session, 0, false));
    }
    SubPlan subplan = planFragmenter.createSubPlans(session, plan, true, WarningCollector.NOOP);
    if (!subplan.getChildren().isEmpty()) {
        throw new AssertionError("Expected subplan to have no children");
    }
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    LocalExecutionPlanner localExecutionPlanner = new LocalExecutionPlanner(metadata, new TypeAnalyzer(sqlParser, metadata), Optional.empty(), pageSourceManager, indexManager, nodePartitioningManager, pageSinkManager, null, expressionCompiler, pageFunctionCompiler, joinFilterFunctionCompiler, new IndexJoinLookupStats(), this.taskManagerConfig, spillerFactory, singleStreamSpillerFactory, partitioningSpillerFactory, new PagesIndex.TestingFactory(false), joinCompiler, new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
    OmniLocalExecutionPlanner omniLocalExecutionPlanner = new OmniLocalExecutionPlanner(localExecutionPlanner);
    ScheduledExecutorService taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s"));
    OutputBuffer outputBuffer = new PartitionedOutputBuffer(new StateMachine<>("bufferState", taskNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(new OutputBuffers.OutputBufferId(0), 0).withNoMoreBufferIds(), new DataSize(1, MEGABYTE), () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), taskNotificationExecutor);
    StageExecutionDescriptor stageExecutionDescriptor = subplan.getFragment().getStageExecutionDescriptor();
    omniLocalExecutionPlanner.plan(taskContext, subplan.getFragment().getRoot(), plan.getTypes(), new PartitioningScheme(Partitioning.create(FIXED_HASH_DISTRIBUTION, ImmutableList.of()), subplan.getFragment().getRoot().getOutputSymbols(), Optional.empty(), false, Optional.of(new int[] { 1 })), stageExecutionDescriptor, subplan.getFragment().getPartitionedSources(), outputBuffer, Optional.empty(), Optional.empty(), null);
}