use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.
the class TestFileSingleStreamSpillerFactory method throwsIfNoDiskSpace.
@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "No free space available for spill")
public void throwsIfNoDiskSpace() {
List<Type> types = ImmutableList.of(BIGINT);
BlockEncodingSerde blockEncodingSerde = new BlockEncodingManager();
List<Path> spillPaths = ImmutableList.of(spillPath1.toPath(), spillPath2.toPath());
FileSingleStreamSpillerFactory spillerFactory = new FileSingleStreamSpillerFactory(// executor won't be closed, because we don't call destroy() on the spiller factory
executor, blockEncodingSerde, new SpillerStats(), spillPaths, 0.0, false, false);
spillerFactory.create(types, new TestingSpillContext(), newSimpleAggregatedMemoryContext().newLocalMemoryContext("test"));
}
use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.
the class TestSqlTaskExecution method testSimple.
@Test(dataProvider = "executionStrategies", timeOut = 20_000)
public void testSimple(PipelineExecutionStrategy executionStrategy) throws Exception {
ScheduledExecutorService taskNotificationExecutor = newScheduledThreadPool(10, threadsNamed("task-notification-%s"));
ScheduledExecutorService driverYieldExecutor = newScheduledThreadPool(2, threadsNamed("driver-yield-%s"));
TaskExecutor taskExecutor = new TaskExecutor(5, 10, 3, 4, TASK_FAIR, Ticker.systemTicker());
taskExecutor.start();
try {
TaskStateMachine taskStateMachine = new TaskStateMachine(TASK_ID, taskNotificationExecutor);
PartitionedOutputBuffer outputBuffer = newTestingOutputBuffer(taskNotificationExecutor);
OutputBufferConsumer outputBufferConsumer = new OutputBufferConsumer(outputBuffer, OUTPUT_BUFFER_ID);
//
// test initialization: simple task with 1 pipeline
//
// pipeline 0 ... pipeline id
// partitioned ... partitioned/unpartitioned pipeline
// grouped ... execution strategy (in grouped test)
// ungrouped ... execution strategy (in ungrouped test)
//
// TaskOutput
// |
// Scan
//
// See #testComplex for all the bahaviors that are tested. Not all of them apply here.
TestingScanOperatorFactory testingScanOperatorFactory = new TestingScanOperatorFactory(0, TABLE_SCAN_NODE_ID, ImmutableList.of(VARCHAR));
TaskOutputOperatorFactory taskOutputOperatorFactory = new TaskOutputOperatorFactory(1, TABLE_SCAN_NODE_ID, outputBuffer, Function.identity(), new PagesSerdeFactory(new BlockEncodingManager(), false));
LocalExecutionPlan localExecutionPlan = new LocalExecutionPlan(ImmutableList.of(new DriverFactory(0, true, true, ImmutableList.of(testingScanOperatorFactory, taskOutputOperatorFactory), OptionalInt.empty(), executionStrategy, Optional.empty())), ImmutableList.of(TABLE_SCAN_NODE_ID), executionStrategy == GROUPED_EXECUTION ? StageExecutionDescriptor.fixedLifespanScheduleGroupedExecution(ImmutableList.of(TABLE_SCAN_NODE_ID), 8) : StageExecutionDescriptor.ungroupedExecution());
TaskContext taskContext = newTestingTaskContext(taskNotificationExecutor, driverYieldExecutor, taskStateMachine);
SqlTaskExecution sqlTaskExecution = SqlTaskExecution.createSqlTaskExecution(taskStateMachine, taskContext, outputBuffer, ImmutableList.of(), localExecutionPlan, taskExecutor, taskNotificationExecutor, createTestSplitMonitor());
//
// test body
assertEquals(taskStateMachine.getState(), TaskState.RUNNING);
switch(executionStrategy) {
case UNGROUPED_EXECUTION:
// add source for pipeline
sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(0, TABLE_SCAN_NODE_ID, Lifespan.taskWide(), 100000, 123)), false)));
// assert that partial task result is produced
outputBufferConsumer.consume(123, ASSERT_WAIT_TIMEOUT);
// pause operator execution to make sure that
// * operatorFactory will be closed even though operator can't execute
// * completedDriverGroups will NOT include the newly scheduled driver group while pause is in place
testingScanOperatorFactory.getPauser().pause();
// add source for pipeline, mark as no more splits
sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(1, TABLE_SCAN_NODE_ID, Lifespan.taskWide(), 200000, 300), newScheduledSplit(2, TABLE_SCAN_NODE_ID, Lifespan.taskWide(), 300000, 200)), true)));
// assert that pipeline will have no more drivers
waitUntilEquals(testingScanOperatorFactory::isOverallNoMoreOperators, true, ASSERT_WAIT_TIMEOUT);
// assert that no DriverGroup is fully completed
assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of());
// resume operator execution
testingScanOperatorFactory.getPauser().resume();
// assert that task result is produced
outputBufferConsumer.consume(300 + 200, ASSERT_WAIT_TIMEOUT);
outputBufferConsumer.assertBufferComplete(ASSERT_WAIT_TIMEOUT);
break;
case GROUPED_EXECUTION:
// add source for pipeline (driver group [1, 5]), mark driver group [1] as noMoreSplits
sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(0, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(1), 0, 1), newScheduledSplit(1, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(5), 100000, 10)), ImmutableSet.of(Lifespan.driverGroup(1)), false)));
// assert that pipeline will have no more drivers for driver group [1]
waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1)), ASSERT_WAIT_TIMEOUT);
// assert that partial result is produced for both driver groups
outputBufferConsumer.consume(1 + 10, ASSERT_WAIT_TIMEOUT);
// assert that driver group [1] is fully completed
waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1)), ASSERT_WAIT_TIMEOUT);
// pause operator execution to make sure that
// * operatorFactory will be closed even though operator can't execute
// * completedDriverGroups will NOT include the newly scheduled driver group while pause is in place
testingScanOperatorFactory.getPauser().pause();
// add source for pipeline (driver group [5]), mark driver group [5] as noMoreSplits
sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(2, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(5), 200000, 300)), ImmutableSet.of(Lifespan.driverGroup(5)), false)));
// assert that pipeline will have no more drivers for driver group [1, 5]
waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)), ASSERT_WAIT_TIMEOUT);
// assert that driver group [5] is NOT YET fully completed
assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1)));
// resume operator execution
testingScanOperatorFactory.getPauser().resume();
// assert that partial result is produced
outputBufferConsumer.consume(300, ASSERT_WAIT_TIMEOUT);
// assert that driver group [1, 5] is fully completed
waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)), ASSERT_WAIT_TIMEOUT);
// pause operator execution to make sure that
// * operatorFactory will be closed even though operator can't execute
// * completedDriverGroups will NOT include the newly scheduled driver group while pause is in place
testingScanOperatorFactory.getPauser().pause();
// add source for pipeline (driver group [7]), mark pipeline as noMoreSplits without explicitly marking driver group 7
sqlTaskExecution.addSources(ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(newScheduledSplit(3, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(7), 300000, 45), newScheduledSplit(4, TABLE_SCAN_NODE_ID, Lifespan.driverGroup(7), 400000, 54)), ImmutableSet.of(), true)));
// assert that pipeline will have no more drivers for driver group [1, 5, 7]
waitUntilEquals(testingScanOperatorFactory::getDriverGroupsWithNoMoreOperators, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5), Lifespan.driverGroup(7)), ASSERT_WAIT_TIMEOUT);
// assert that pipeline will have no more drivers
waitUntilEquals(testingScanOperatorFactory::isOverallNoMoreOperators, true, ASSERT_WAIT_TIMEOUT);
// assert that driver group [1, 5] is fully completed
assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)));
// resume operator execution
testingScanOperatorFactory.getPauser().resume();
// assert driver group [7] is not completed before output buffer is consumed
MILLISECONDS.sleep(1000);
assertEquals(taskContext.getCompletedDriverGroups(), ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5)));
// assert that result is produced
outputBufferConsumer.consume(45 + 54, ASSERT_WAIT_TIMEOUT);
outputBufferConsumer.assertBufferComplete(ASSERT_WAIT_TIMEOUT);
// assert that driver group [1, 5, 7] is fully completed
waitUntilEquals(taskContext::getCompletedDriverGroups, ImmutableSet.of(Lifespan.driverGroup(1), Lifespan.driverGroup(5), Lifespan.driverGroup(7)), ASSERT_WAIT_TIMEOUT);
break;
default:
throw new UnsupportedOperationException();
}
// complete the task by calling abort on it
outputBufferConsumer.abort();
TaskState taskState = taskStateMachine.getStateChange(TaskState.RUNNING).get(10, SECONDS);
assertEquals(taskState, TaskState.FINISHED);
} finally {
taskExecutor.stop();
taskNotificationExecutor.shutdownNow();
driverYieldExecutor.shutdown();
}
}
use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
MetadataManager metadata = MetadataManager.createTestMetadataManager();
PageSourceManager pageSourceManager = new PageSourceManager();
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager();
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager, new NodeSelectionStats());
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
return new LocalExecutionPlanner(metadata, Optional.empty(), pageSourceManager, new IndexManager(), partitioningProviderManager, nodePartitioningManager, new PageSinkManager(), new ConnectorMetadataUpdaterManager(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new MemoryManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new BlockEncodingManager(), new PagesIndex.TestingFactory(false), new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), new LookupJoinOperators(), new OrderingCompiler(), jsonCodec(TableCommitContext.class), new RowExpressionDeterminismEvaluator(metadata), new NoOpFragmentResultCacheManager(), new ObjectMapper(), (session) -> {
throw new UnsupportedOperationException();
});
}
use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.
the class TestBlockEncodingBuffers method serialize.
private static Block serialize(BlockEncodingBuffer buffer) {
SliceOutput output = new DynamicSliceOutput(toIntExact(buffer.getSerializedSizeInBytes()));
buffer.serializeTo(output);
BlockEncodingManager blockEncodingSerde = new BlockEncodingManager();
return readBlock(blockEncodingSerde, output.slice().getInput());
}
use of com.facebook.presto.common.block.BlockEncodingManager in project presto by prestodb.
the class TestOptimizedPartitionedOutputOperator method createOptimizedPartitionedOutputOperator.
private OptimizedPartitionedOutputOperator createOptimizedPartitionedOutputOperator(List<Type> types, List<Integer> partitionChannel, PartitionFunction partitionFunction, PartitionedOutputBuffer buffer, OptionalInt nullChannel, DataSize maxMemory) {
PagesSerdeFactory serdeFactory = new PagesSerdeFactory(new BlockEncodingManager(), false);
OutputPartitioning outputPartitioning = new OutputPartitioning(partitionFunction, partitionChannel, ImmutableList.of(Optional.empty()), false, nullChannel);
OptimizedPartitionedOutputFactory operatorFactory = new OptimizedPartitionedOutputFactory(buffer, maxMemory);
return (OptimizedPartitionedOutputOperator) operatorFactory.createOutputOperator(0, new PlanNodeId("plan-node-0"), types, Function.identity(), Optional.of(outputPartitioning), serdeFactory).createOperator(createDriverContext());
}
Aggregations