use of io.trino.util.FinalizerService in project trino by trinodb.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
PageSourceManager pageSourceManager = new PageSourceManager();
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(PLANNER_CONTEXT.getTypeOperators());
NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)));
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, blockTypeOperators);
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(PLANNER_CONTEXT.getFunctionManager(), 0);
return new LocalExecutionPlanner(PLANNER_CONTEXT, createTestingTypeAnalyzer(PLANNER_CONTEXT), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockDirectExchangeClientSupplier(), new ExpressionCompiler(PLANNER_CONTEXT.getFunctionManager(), pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(PLANNER_CONTEXT.getFunctionManager()), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new PagesIndex.TestingFactory(false), new JoinCompiler(PLANNER_CONTEXT.getTypeOperators()), new TrinoOperatorFactories(), new OrderingCompiler(PLANNER_CONTEXT.getTypeOperators()), new DynamicFilterConfig(), blockTypeOperators, new TableExecuteContextManager(), new ExchangeManagerRegistry(new ExchangeHandleResolver()));
}
use of io.trino.util.FinalizerService in project trino by trinodb.
the class TestLocalExchange method setUp.
@BeforeMethod
public void setUp() {
NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(new FinalizerService())));
nodePartitioningManager = new NodePartitioningManager(nodeScheduler, new BlockTypeOperators(new TypeOperators()));
}
use of io.trino.util.FinalizerService in project trino by trinodb.
the class TestNodeScheduler method setUp.
@BeforeMethod
public void setUp() {
session = TestingSession.testSessionBuilder().build();
finalizerService = new FinalizerService();
nodeTaskMap = new NodeTaskMap(finalizerService);
nodeManager = new InMemoryNodeManager();
nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(nodeManager, nodeSchedulerConfig, nodeTaskMap));
// contents of taskMap indicate the node-task map for the current stage
taskMap = new HashMap<>();
nodeSelector = nodeScheduler.createNodeSelector(session, Optional.of(CONNECTOR_ID));
remoteTaskExecutor = newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s"));
remoteTaskScheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s"));
finalizerService.start();
}
use of io.trino.util.FinalizerService in project trino by trinodb.
the class TestSqlStage method testFinalStageInfoInternal.
private void testFinalStageInfoInternal() throws Exception {
NodeTaskMap nodeTaskMap = new NodeTaskMap(new FinalizerService());
StageId stageId = new StageId(new QueryId("query"), 0);
SqlStage stage = createSqlStage(stageId, createExchangePlanFragment(), ImmutableMap.of(), new MockRemoteTaskFactory(executor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, executor, new SplitSchedulerStats());
// add listener that fetches stage info when the final status is available
SettableFuture<StageInfo> finalStageInfo = SettableFuture.create();
stage.addFinalStageInfoListener(finalStageInfo::set);
// in a background thread add a ton of tasks
CountDownLatch latch = new CountDownLatch(1000);
Future<?> addTasksTask = executor.submit(() -> {
try {
for (int i = 0; i < 1_000_000; i++) {
if (Thread.interrupted()) {
return;
}
InternalNode node = new InternalNode("source" + i, URI.create("http://10.0.0." + (i / 10_000) + ":" + (i % 10_000)), NodeVersion.UNKNOWN, false);
stage.createTask(node, i, 0, Optional.empty(), createInitialEmptyOutputBuffers(ARBITRARY), ImmutableMultimap.of(), ImmutableMultimap.of(), ImmutableSet.of());
latch.countDown();
}
} finally {
while (latch.getCount() > 0) {
latch.countDown();
}
}
});
// wait for some tasks to be created, and then abort the query
latch.await(1, MINUTES);
assertFalse(stage.getStageInfo().getTasks().isEmpty());
stage.finish();
// once the final stage info is available, verify that it is complete
StageInfo stageInfo = finalStageInfo.get(1, MINUTES);
assertFalse(stageInfo.getTasks().isEmpty());
assertTrue(stageInfo.isCompleteInfo());
assertSame(stage.getStageInfo(), stageInfo);
// cancel the background thread adding tasks
addTasksTask.cancel(true);
}
use of io.trino.util.FinalizerService in project trino by trinodb.
the class TestFaultTolerantStageScheduler method beforeClass.
@BeforeClass
public void beforeClass() {
finalizerService = new FinalizerService();
finalizerService.start();
nodeTaskMap = new NodeTaskMap(finalizerService);
}
Aggregations