Examples with FinalizerService com.facebook.presto.util.FinalizerService used on opensource projects

Example 1 with FinalizerService

use of com.facebook.presto.util.FinalizerService in project presto by prestodb.

the class TestNodeScheduler method setUp.

@BeforeMethod
public void setUp() {
    session = TestingSession.testSessionBuilder().build();
    finalizerService = new FinalizerService();
    nodeTaskMap = new NodeTaskMap(finalizerService);
    nodeManager = new InMemoryNodeManager();
    ImmutableList.Builder<InternalNode> nodeBuilder = ImmutableList.builder();
    nodeBuilder.add(new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false));
    nodeBuilder.add(new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false));
    nodeBuilder.add(new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false));
    List<InternalNode> nodes = nodeBuilder.build();
    nodeManager.addNode(CONNECTOR_ID, nodes);
    nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    // contents of taskMap indicate the node-task map for the current stage
    taskMap = new HashMap<>();
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    remoteTaskExecutor = newCachedThreadPool(daemonThreadsNamed("remoteTaskExecutor-%s"));
    remoteTaskScheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("remoteTaskScheduledExecutor-%s"));
    finalizerService.start();
}

Example 2 with FinalizerService

use of com.facebook.presto.util.FinalizerService in project presto by prestodb.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager();
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager, new NodeSelectionStats());
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    return new LocalExecutionPlanner(metadata, Optional.empty(), pageSourceManager, new IndexManager(), partitioningProviderManager, nodePartitioningManager, new PageSinkManager(), new ConnectorMetadataUpdaterManager(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new MemoryManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new BlockEncodingManager(), new PagesIndex.TestingFactory(false), new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), new LookupJoinOperators(), new OrderingCompiler(), jsonCodec(TableCommitContext.class), new RowExpressionDeterminismEvaluator(metadata), new NoOpFragmentResultCacheManager(), new ObjectMapper(), (session) -> {
        throw new UnsupportedOperationException();
    });
}

Example 3 with FinalizerService

use of com.facebook.presto.util.FinalizerService in project presto by prestodb.

the class TestCostCalculator method setUp.

@BeforeClass
public void setUp() {
    TaskCountEstimator taskCountEstimator = new TaskCountEstimator(() -> NUMBER_OF_NODES);
    costCalculatorUsingExchanges = new CostCalculatorUsingExchanges(taskCountEstimator);
    costCalculatorWithEstimatedExchanges = new CostCalculatorWithEstimatedExchanges(costCalculatorUsingExchanges, taskCountEstimator);
    session = testSessionBuilder().setCatalog("tpch").build();
    CatalogManager catalogManager = new CatalogManager();
    catalogManager.registerCatalog(createBogusTestingCatalog("tpch"));
    transactionManager = createTestTransactionManager(catalogManager);
    metadata = createTestMetadataManager(transactionManager, new FeaturesConfig());
    finalizerService = new FinalizerService();
    finalizerService.start();
    nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager();
    nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager, new NodeSelectionStats());
    planFragmenter = new PlanFragmenter(metadata, nodePartitioningManager, new QueryManagerConfig(), new SqlParser(), new FeaturesConfig());
}

Example 4 with FinalizerService

use of com.facebook.presto.util.FinalizerService in project presto by prestodb.

the class TestSqlStageExecution method testFinalStageInfoInternal.

private void testFinalStageInfoInternal() throws Exception {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(new FinalizerService());
    StageId stageId = new StageId(new QueryId("query"), 0);
    SqlStageExecution stage = createSqlStageExecution(new StageExecutionId(stageId, 0), createExchangePlanFragment(), new MockRemoteTaskFactory(executor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, executor, new NoOpFailureDetector(), new SplitSchedulerStats(), new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty()));
    stage.setOutputBuffers(createInitialEmptyOutputBuffers(ARBITRARY));
    // add listener that fetches stage info when the final status is available
    SettableFuture<StageExecutionInfo> finalStageInfo = SettableFuture.create();
    stage.addFinalStageInfoListener(finalStageInfo::set);
    // in a background thread add a ton of tasks
    CountDownLatch latch = new CountDownLatch(1000);
    Future<?> addTasksTask = executor.submit(() -> {
        try {
            for (int i = 0; i < 1_000_000; i++) {
                if (Thread.interrupted()) {
                    return;
                }
                InternalNode node = new InternalNode("source" + i, URI.create("http://10.0.0." + (i / 10_000) + ":" + (i % 10_000)), NodeVersion.UNKNOWN, false);
                stage.scheduleTask(node, i);
                latch.countDown();
            }
        } finally {
            while (latch.getCount() > 0) {
                latch.countDown();
            }
        }
    });
    // wait for some tasks to be created, and then abort the query
    latch.await(1, MINUTES);
    assertFalse(stage.getStageExecutionInfo().getTasks().isEmpty());
    stage.abort();
    // once the final stage info is available, verify that it is complete
    StageExecutionInfo stageInfo = finalStageInfo.get(1, MINUTES);
    assertFalse(stageInfo.getTasks().isEmpty());
    assertTrue(stageInfo.isFinal());
    assertSame(stage.getStageExecutionInfo(), stageInfo);
    // cancel the background thread adding tasks
    addTasksTask.cancel(true);
}