Search in sources :

Example 6 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestFileSingleStreamSpillerFactory method throwsIfNoDiskSpace.

@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "No free space available for spill")
public void throwsIfNoDiskSpace() {
    List<Type> types = ImmutableList.of(BIGINT);
    BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager();
    List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath());
    FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
    executor, blockEncodingSerde, new SpillerStats(), spillPaths, 0.0, false, false);
    spillerFactory.create(types, new TestingSpillContext(), newSimpleAggregatedMemoryContext().newLocalMemoryContext("test"));
}
Also used : Path(java.nio.file.Path) Type(com.facebook.presto.common.type.Type) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) BlockEncodingSerde(com.facebook.presto.common.block.BlockEncodingSerde) Test(org.testng.annotations.Test)

Example 7 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestSqlTaskExecution method testSimple.

@Test(dataProvider = "executionStrategies", timeOut = 20_000)
public void testSimple(PipelineExecutionStrategy executionStrategy) throws Exception {
    ScheduledExecutorService taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s"));
    ScheduledExecutorService driverYieldExecutor = newScheduledThreadPool(2, threadsNamed("driver-yield-%s"));
    TaskExecutor taskExecutor = new TaskExecutor(5, 10, 3, 4, TASK_FAIR, Ticker.systemTicker());
    taskExecutor.start();
    try {
        TaskStateMachine taskStateMachine = new TaskStateMachine(TASK_ID, taskNotificationExecutor);
        PartitionedOutputBuffer outputBuffer = newTestingOutputBuffer(taskNotificationExecutor);
        OutputBufferConsumer outputBufferConsumer = new OutputBufferConsumer(outputBuffer, OUTPUT_BUFFER_ID);
        // 
        // test initialization: simple task with 1 pipeline
        // 
        // pipeline 0  ... pipeline id
        // partitioned ... partitioned/unpartitioned pipeline
        // grouped   ... execution strategy (in grouped test)
        // ungrouped  ... execution strategy (in ungrouped test)
        // 
        // TaskOutput
        // |
        // Scan
        // 
        // See #testComplex for all the bahaviors that are tested. Not all of them apply here.
        TestingScanOperatorFactory testingScanOperatorFactory = new TestingScanOperatorFactory(0, TABLE_SCAN_NODE_ID, ImmutableList.of(VARCHAR));
        TaskOutputOperatorFactory taskOutputOperatorFactory = new TaskOutputOperatorFactory(1, TABLE_SCAN_NODE_ID, outputBuffer, Function.identity(), new PagesSerdeFactory(new BlockEncodingManager(), false));
        LocalExecutionPlan localExecutionPlan = new LocalExecutionPlan(ImmutableList.of(new DriverFactory(0, true, true, ImmutableList.of(testingScanOperatorFactory, taskOutputOperatorFactory), OptionalInt.empty(), executionStrategy, Optional.empty())), ImmutableList.of(TABLE_SCAN_NODE_ID), executionStrategy == GROUPED_EXECUTION ? StageExecutionDescriptor.fixedLifespanScheduleGroupedExecution(ImmutableList.of(TABLE_SCAN_NODE_ID), 8) : StageExecutionDescriptor.ungroupedExecution());
        TaskContext taskContext = newTestingTaskContext(taskNotificationExecutor, driverYieldExecutor, taskStateMachine);
        SqlTaskExecution sqlTaskExecution = SqlTaskExecution.createSqlTaskExecution(taskStateMachine, taskContext, outputBuffer, ImmutableList.of(), localExecutionPlan, taskExecutor, taskNotificationExecutor, createTestSplitMonitor());
        // 
        // test body
        assertEquals(taskStateMachine.getState(), TaskState.RUNNING);
        switch(executionStrategy) {
            case UNGROUPED_EXECUTION:
                // add source for pipeline
                sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(0, TABLE_SCAN_NODE_ID, Lifespan.taskWide(), 100000, 123)), false)));
                // assert that partial task result is produced
                outputBufferConsumer.consume(123, ASSERT_WAIT_TIMEOUT);
                // pause operator execution to make sure that
                // * operatorFactory will be closed even though operator can't execute
                // * completedDriverGroups will NOT include the newly scheduled driver group while pause is in place
                testingScanOperatorFactory.getPauser().pause();
                // add source for pipeline, mark as no more splits
                sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(1, TABLE_SCAN_NODE_ID, Lifespan.taskWide(), 200000, 300), newScheduledSplit(2, TABLE_SCAN_NODE_ID, Lifespan.taskWide(), 300000, 200)), true)));
                // assert that pipeline will have no more drivers
                waitUntilEquals(testingScanOperatorFactory::isOverallNoMoreOperators, true, ASSERT_WAIT_TIMEOUT);
                // assert that no DriverGroup is fully completed
                assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of());
                // resume operator execution
                testingScanOperatorFactory.getPauser().resume();
                // assert that task result is produced
                outputBufferConsumer.consume(300 + 200, ASSERT_WAIT_TIMEOUT);
                outputBufferConsumer.assertBufferComplete(ASSERT_WAIT_TIMEOUT);
                break;
            case GROUPED_EXECUTION:
                // add source for pipeline (driver group [1, 5]), mark driver group [1] as noMoreSplits
                sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(0, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(1), 0, 1), newScheduledSplit(1, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(5), 100000, 10)), ImmutableSet.of(Lifespan.driverGroup(1)), false)));
                // assert that pipeline will have no more drivers for driver group [1]
                waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1)), ASSERT_WAIT_TIMEOUT);
                // assert that partial result is produced for both driver groups
                outputBufferConsumer.consume(1 + 10, ASSERT_WAIT_TIMEOUT);
                // assert that driver group [1] is fully completed
                waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1)), ASSERT_WAIT_TIMEOUT);
                // pause operator execution to make sure that
                // * operatorFactory will be closed even though operator can't execute
                // * completedDriverGroups will NOT include the newly scheduled driver group while pause is in place
                testingScanOperatorFactory.getPauser().pause();
                // add source for pipeline (driver group [5]), mark driver group [5] as noMoreSplits
                sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(2, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(5), 200000, 300)), ImmutableSet.of(Lifespan.driverGroup(5)), false)));
                // assert that pipeline will have no more drivers for driver group [1, 5]
                waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)), ASSERT_WAIT_TIMEOUT);
                // assert that driver group [5] is NOT YET fully completed
                assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1)));
                // resume operator execution
                testingScanOperatorFactory.getPauser().resume();
                // assert that partial result is produced
                outputBufferConsumer.consume(300, ASSERT_WAIT_TIMEOUT);
                // assert that driver group [1, 5] is fully completed
                waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)), ASSERT_WAIT_TIMEOUT);
                // pause operator execution to make sure that
                // * operatorFactory will be closed even though operator can't execute
                // * completedDriverGroups will NOT include the newly scheduled driver group while pause is in place
                testingScanOperatorFactory.getPauser().pause();
                // add source for pipeline (driver group [7]), mark pipeline as noMoreSplits without explicitly marking driver group 7
                sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(3, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(7), 300000, 45), newScheduledSplit(4, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(7), 400000, 54)), ImmutableSet.of(), true)));
                // assert that pipeline will have no more drivers for driver group [1, 5, 7]
                waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5), Lifespan.driverGroup(7)), ASSERT_WAIT_TIMEOUT);
                // assert that pipeline will have no more drivers
                waitUntilEquals(testingScanOperatorFactory::isOverallNoMoreOperators, true, ASSERT_WAIT_TIMEOUT);
                // assert that driver group [1, 5] is fully completed
                assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)));
                // resume operator execution
                testingScanOperatorFactory.getPauser().resume();
                // assert driver group [7] is not completed before output buffer is consumed
                MILLISECONDS.sleep(1000);
                assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)));
                // assert that result is produced
                outputBufferConsumer.consume(45 + 54, ASSERT_WAIT_TIMEOUT);
                outputBufferConsumer.assertBufferComplete(ASSERT_WAIT_TIMEOUT);
                // assert that driver group [1, 5, 7] is fully completed
                waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5), Lifespan.driverGroup(7)), ASSERT_WAIT_TIMEOUT);
                break;
            default:
                throw new UnsupportedOperationException();
        }
        // complete the task by calling abort on it
        outputBufferConsumer.abort();
        TaskState taskState = taskStateMachine.getStateChange(TaskState.RUNNING).get(10, SECONDS);
        assertEquals(taskState, TaskState.FINISHED);
    } finally {
        taskExecutor.stop();
        taskNotificationExecutor.shutdownNow();
        driverYieldExecutor.shutdown();
    }
}
Also used : ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) PartitionedOutputBuffer(com.facebook.presto.execution.buffer.PartitionedOutputBuffer) TaskContext(com.facebook.presto.operator.TaskContext) LocalExecutionPlan(com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) PagesSerdeFactory(com.facebook.presto.execution.buffer.PagesSerdeFactory) TaskExecutor(com.facebook.presto.execution.executor.TaskExecutor) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) DriverFactory(com.facebook.presto.operator.DriverFactory) TaskOutputOperatorFactory(com.facebook.presto.operator.TaskOutputOperator.TaskOutputOperatorFactory) Test(org.testng.annotations.Test)

Example 8 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager();
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager, new NodeSelectionStats());
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    return new LocalExecutionPlanner(metadata, Optional.empty(), pageSourceManager, new IndexManager(), partitioningProviderManager, nodePartitioningManager, new PageSinkManager(), new ConnectorMetadataUpdaterManager(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new MemoryManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new BlockEncodingManager(), new PagesIndex.TestingFactory(false), new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), new LookupJoinOperators(), new OrderingCompiler(), jsonCodec(TableCommitContext.class), new RowExpressionDeterminismEvaluator(metadata), new NoOpFragmentResultCacheManager(), new ObjectMapper(), (session) -> {
        throw new UnsupportedOperationException();
    });
}
Also used : WarningCollector(com.facebook.presto.spi.WarningCollector) LocalExecutionPlanner(com.facebook.presto.sql.planner.LocalExecutionPlanner) MetadataManager(com.facebook.presto.metadata.MetadataManager) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) EventListenerManager(com.facebook.presto.eventlistener.EventListenerManager) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) TableWriteInfo(com.facebook.presto.execution.scheduler.TableWriteInfo) PagesIndex(com.facebook.presto.operator.PagesIndex) StageExecutionDescriptor(com.facebook.presto.operator.StageExecutionDescriptor) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) NoOpFragmentResultCacheManager(com.facebook.presto.operator.NoOpFragmentResultCacheManager) FinalizerService(com.facebook.presto.util.FinalizerService) JsonObjectMapperProvider(com.facebook.airlift.json.JsonObjectMapperProvider) PartitioningScheme(com.facebook.presto.sql.planner.PartitioningScheme) URI(java.net.URI) PageFunctionCompiler(com.facebook.presto.sql.gen.PageFunctionCompiler) ConnectorMetadataUpdaterManager(com.facebook.presto.metadata.ConnectorMetadataUpdaterManager) ImmutableSet(com.google.common.collect.ImmutableSet) PageSinkManager(com.facebook.presto.split.PageSinkManager) ImmutableMap(com.google.common.collect.ImmutableMap) ResourceGroupId(com.facebook.presto.spi.resourceGroups.ResourceGroupId) JsonCodec.jsonCodec(com.facebook.airlift.json.JsonCodec.jsonCodec) SOURCE_DISTRIBUTION(com.facebook.presto.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) List(java.util.List) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) AllowAllAccessControl(com.facebook.presto.security.AllowAllAccessControl) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) OutputBuffers(com.facebook.presto.execution.buffer.OutputBuffers) SINGLE_DISTRIBUTION(com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) RowExpressionDeterminismEvaluator(com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator) LookupJoinOperators(com.facebook.presto.operator.LookupJoinOperators) NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) TestingSplit(com.facebook.presto.testing.TestingSplit) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) ImmutableList(com.google.common.collect.ImmutableList) PageSourceManager(com.facebook.presto.split.PageSourceManager) OrderingCompiler(com.facebook.presto.sql.gen.OrderingCompiler) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) GenericSpillerFactory(com.facebook.presto.spiller.GenericSpillerFactory) IndexManager(com.facebook.presto.index.IndexManager) TableHandle(com.facebook.presto.spi.TableHandle) MemoryManagerConfig(com.facebook.presto.memory.MemoryManagerConfig) PlanFragmentId(com.facebook.presto.sql.planner.plan.PlanFragmentId) IndexJoinLookupStats(com.facebook.presto.operator.index.IndexJoinLookupStats) TransactionManager(com.facebook.presto.transaction.TransactionManager) InMemoryNodeManager(com.facebook.presto.metadata.InMemoryNodeManager) Partitioning(com.facebook.presto.sql.planner.Partitioning) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) NodePartitioningManager(com.facebook.presto.sql.planner.NodePartitioningManager) TableCommitContext(com.facebook.presto.operator.TableCommitContext) SplitMonitor(com.facebook.presto.event.SplitMonitor) Executor(java.util.concurrent.Executor) Session(com.facebook.presto.Session) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) JoinFilterFunctionCompiler(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler) TestingTransactionHandle(com.facebook.presto.testing.TestingTransactionHandle) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) TestingColumnHandle(com.facebook.presto.testing.TestingMetadata.TestingColumnHandle) TableScanNode(com.facebook.presto.spi.plan.TableScanNode) StatsAndCosts(com.facebook.presto.cost.StatsAndCosts) Split(com.facebook.presto.metadata.Split) TestingTableHandle(com.facebook.presto.testing.TestingMetadata.TestingTableHandle) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) PartitioningProviderManager(com.facebook.presto.sql.planner.PartitioningProviderManager) PageFunctionCompiler(com.facebook.presto.sql.gen.PageFunctionCompiler) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) PartitioningProviderManager(com.facebook.presto.sql.planner.PartitioningProviderManager) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) TableCommitContext(com.facebook.presto.operator.TableCommitContext) PagesIndex(com.facebook.presto.operator.PagesIndex) PageSourceManager(com.facebook.presto.split.PageSourceManager) NodePartitioningManager(com.facebook.presto.sql.planner.NodePartitioningManager) NoOpFragmentResultCacheManager(com.facebook.presto.operator.NoOpFragmentResultCacheManager) OrderingCompiler(com.facebook.presto.sql.gen.OrderingCompiler) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) GenericSpillerFactory(com.facebook.presto.spiller.GenericSpillerFactory) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) PageSinkManager(com.facebook.presto.split.PageSinkManager) LookupJoinOperators(com.facebook.presto.operator.LookupJoinOperators) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) RowExpressionDeterminismEvaluator(com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator) LocalExecutionPlanner(com.facebook.presto.sql.planner.LocalExecutionPlanner) IndexJoinLookupStats(com.facebook.presto.operator.index.IndexJoinLookupStats) JoinFilterFunctionCompiler(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler) MemoryManagerConfig(com.facebook.presto.memory.MemoryManagerConfig) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) InMemoryNodeManager(com.facebook.presto.metadata.InMemoryNodeManager) IndexManager(com.facebook.presto.index.IndexManager) NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) MetadataManager(com.facebook.presto.metadata.MetadataManager) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) FinalizerService(com.facebook.presto.util.FinalizerService) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) ConnectorMetadataUpdaterManager(com.facebook.presto.metadata.ConnectorMetadataUpdaterManager) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler)

Example 9 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestBlockEncodingBuffers method serialize.

private static Block serialize(BlockEncodingBuffer buffer) {
    SliceOutput output = new DynamicSliceOutput(toIntExact(buffer.getSerializedSizeInBytes()));
    buffer.serializeTo(output);
    BlockEncodingManager blockEncodingSerde = new BlockEncodingManager();
    return readBlock(blockEncodingSerde, output.slice().getInput());
}
Also used : SliceOutput(io.airlift.slice.SliceOutput) DynamicSliceOutput(io.airlift.slice.DynamicSliceOutput) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) DynamicSliceOutput(io.airlift.slice.DynamicSliceOutput)

Example 10 with BlockEncodingManager

use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.

the class TestOptimizedPartitionedOutputOperator method createOptimizedPartitionedOutputOperator.

private OptimizedPartitionedOutputOperator createOptimizedPartitionedOutputOperator(List<Type> types, List<Integer> partitionChannel, PartitionFunction partitionFunction, PartitionedOutputBuffer buffer, OptionalInt nullChannel, DataSize maxMemory) {
    PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(), false);
    OutputPartitioning outputPartitioning = new OutputPartitioning(partitionFunction, partitionChannel, ImmutableList.of(Optional.empty()), false, nullChannel);
    OptimizedPartitionedOutputFactory operatorFactory = new OptimizedPartitionedOutputFactory(buffer, maxMemory);
    return (OptimizedPartitionedOutputOperator) operatorFactory.createOutputOperator(0, new PlanNodeId("plan-node-0"), types, Function.identity(), Optional.of(outputPartitioning), serdeFactory).createOperator(createDriverContext());
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) PagesSerdeFactory(com.facebook.presto.execution.buffer.PagesSerdeFactory) OptimizedPartitionedOutputFactory(com.facebook.presto.operator.repartition.OptimizedPartitionedOutputOperator.OptimizedPartitionedOutputFactory) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) OutputPartitioning(com.facebook.presto.sql.planner.OutputPartitioning)

Aggregations

BlockEncodingManager (com.facebook.presto.common.block.BlockEncodingManager)17 Test (org.testng.annotations.Test)7 Page (com.facebook.presto.common.Page)5 PagesSerdeFactory (com.facebook.presto.execution.buffer.PagesSerdeFactory)5 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)5 BlockEncodingSerde (com.facebook.presto.common.block.BlockEncodingSerde)4 Type (com.facebook.presto.common.type.Type)3 PartitionedOutputBuffer (com.facebook.presto.execution.buffer.PartitionedOutputBuffer)3 TaskExecutor (com.facebook.presto.execution.executor.TaskExecutor)3 FeaturesConfig (com.facebook.presto.sql.analyzer.FeaturesConfig)3 Path (java.nio.file.Path)3 OutputBuffers (com.facebook.presto.execution.buffer.OutputBuffers)2 LocalMemoryContext (com.facebook.presto.memory.context.LocalMemoryContext)2 SerializedPage (com.facebook.presto.spi.page.SerializedPage)2 DynamicSliceOutput (io.airlift.slice.DynamicSliceOutput)2 DataSize (io.airlift.units.DataSize)2 JsonCodec.jsonCodec (com.facebook.airlift.json.JsonCodec.jsonCodec)1 JsonObjectMapperProvider (com.facebook.airlift.json.JsonObjectMapperProvider)1 Session (com.facebook.presto.Session)1 TEST_SESSION (com.facebook.presto.SessionTestUtils.TEST_SESSION)1