Search in sources :

Example 11 with SimpleTtlNodeSelectorConfig

use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.

the class TestNodeScheduler method testAffinityAssignmentWithConsistentHashingWithVirtualNodes.

@Test
public void testAffinityAssignmentWithConsistentHashingWithVirtualNodes() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setNodeSelectionHashStrategy(CONSISTENT_HASHING).setMinVirtualNodeCount(5).setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    Set<Split> splits = new HashSet<>();
    IntStream.range(0, 10).forEach(i -> splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(i))));
    InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
    InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
    InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
    InternalNode node4 = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
    // In setup node 1-3 are added to node manager
    // consistent hashing ring for nodes:
    // entry0 (-1907319920): node2
    // entry1 (-1028466245): node3
    // entry2 ( -546736344): node2
    // entry3 ( 1127574531): node3
    // entry4 ( 1166245243): node1
    // entry5 ( 2145381619): node1
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    // hashing value for splits:
    // 0: -1962219106 -> entry0 -> node2
    // 1:   145569539 -> entry3 -> node3
    // 2: -1599101205 -> entry1 -> node3
    // 3:  -165119218 -> entry3 -> node3
    // 4:  1142216720 -> entry4 -> node1
    // 5:  1347620135 -> entry5 -> node1
    // 6:  1232195252 -> entry5 -> node1
    // 7:   427886318 -> entry3 -> node3
    // 8:  1469878697 -> entry5 -> node1
    // 9:   296801082 -> entry3 -> node3
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 3);
    // node1: split 4, 5, 6, 8
    Collection<ConnectorSplit> node1Splits = splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
    // node2: split 0
    Collection<Object> node2Splits = splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
    // node3: split 1, 2, 3, 7, 9
    Collection<Object> node3Splits = splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
    // Scheduling the same splits on the same set of nodes should give the same assignment
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    assertEquals(splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node1Splits);
    assertEquals(splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node2Splits);
    assertEquals(splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node3Splits);
    // Adding node4, consistent hashing ring for nodes:
    // entry0 (-1907319920): node2
    // entry1 (-1616890413): node4
    // entry2 (-1028466245): node3
    // entry3 ( -546736344): node2
    // entry4 ( 1127574531): node3
    // entry5 ( 1166245243): node1
    // entry6 ( 1691928386): node4
    // entry7 ( 2145381619): node1
    nodeManager.addNode(CONNECTOR_ID, node4);
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    // hashing value for splits:
    // 0: -1962219106 -> entry0 -> node2
    // 1:   145569539 -> entry4 -> node3
    // 2: -1599101205 -> entry2 -> node3
    // 3:  -165119218 -> entry4 -> node3
    // 4:  1142216720 -> entry5 -> node1
    // 5:  1347620135 -> entry6 -> node4
    // 6:  1232195252 -> entry6 -> node4
    // 7:   427886318 -> entry4 -> node3
    // 8:  1469878697 -> entry6 -> node4
    // 9:   296801082 -> entry4 -> node3
    assertEquals(splitPlacementResult.getAssignments().keySet().size(), 4);
    assertEquals(splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(4));
    assertEquals(splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node2Splits);
    assertEquals(splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(1, 2, 3, 7, 9));
    assertEquals(splitPlacementResult.getAssignments().get(node4).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(5, 6, 8));
}
Also used : NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) TestingTransactionHandle(com.facebook.presto.testing.TestingTransactionHandle) NodeSelector(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelector) InternalNode(com.facebook.presto.metadata.InternalNode) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 12 with SimpleTtlNodeSelectorConfig

use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.

the class TestNodeScheduler method testHardAffinityAssignment.

@Test
public void testHardAffinityAssignment() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
    Set<Split> splits = new HashSet<>();
    // Adding one more split (1 % 3 = 1), 1 splits will be distributed to 1 nodes
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
    splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
    SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
    for (Split split : splitPlacementResult.getAssignments().values()) {
        assertTrue(split.getSplitContext().isCacheable());
    }
}
Also used : NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) LegacyNetworkTopology(com.facebook.presto.execution.scheduler.LegacyNetworkTopology) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) TestingTransactionHandle(com.facebook.presto.testing.TestingTransactionHandle) NodeSelector(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelector) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 13 with SimpleTtlNodeSelectorConfig

use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.

the class TestSimpleTtlNodeSelectorConfig method testExplicitPropertyMappings.

@Test
public void testExplicitPropertyMappings() {
    Map<String, String> properties = new ImmutableMap.Builder<String, String>().put("simple-ttl-node-selector.use-default-execution-time-estimate-as-fallback", "true").put("simple-ttl-node-selector.default-execution-time-estimate", "1h").build();
    SimpleTtlNodeSelectorConfig expected = new SimpleTtlNodeSelectorConfig().setUseDefaultExecutionTimeEstimateAsFallback(true).setDefaultExecutionTimeEstimate(new Duration(1, TimeUnit.HOURS));
    ConfigAssertions.assertFullMapping(properties, expected);
}
Also used : Duration(io.airlift.units.Duration) ImmutableMap(com.google.common.collect.ImmutableMap) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) Test(org.testng.annotations.Test)

Example 14 with SimpleTtlNodeSelectorConfig

use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.

the class TestSourcePartitionedScheduler method getSourcePartitionedScheduler.

private static StageScheduler getSourcePartitionedScheduler(ConnectorSplitSource connectorSplitSource, SqlStageExecution stage, InternalNodeManager nodeManager, NodeTaskMap nodeTaskMap, int splitBatchSize) {
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setIncludeCoordinator(false).setMaxSplitsPerNode(20).setMaxPendingSplitsPerTask(0);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
    SplitSource splitSource = new ConnectorAwareSplitSource(CONNECTOR_ID, TestingTransactionHandle.create(), connectorSplitSource);
    SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(TestingSession.testSessionBuilder().build(), splitSource.getConnectorId()), stage::getAllTasks);
    return newSourcePartitionedSchedulerAsStageScheduler(stage, TABLE_SCAN_NODE_ID, splitSource, placementPolicy, splitBatchSize);
}
Also used : NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) SplitSource(com.facebook.presto.split.SplitSource) ConnectorAwareSplitSource(com.facebook.presto.split.ConnectorAwareSplitSource) FixedSplitSource(com.facebook.presto.spi.FixedSplitSource) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) ConnectorAwareSplitSource(com.facebook.presto.split.ConnectorAwareSplitSource)

Example 15 with SimpleTtlNodeSelectorConfig

use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.

the class TestSourcePartitionedScheduler method testNoNodes.

@Test
public void testNoNodes() {
    try {
        NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
        InMemoryNodeManager nodeManager = new InMemoryNodeManager();
        NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(false), nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
        SubPlan plan = createPlan();
        SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
        StageScheduler scheduler = newSourcePartitionedSchedulerAsStageScheduler(stage, TABLE_SCAN_NODE_ID, new ConnectorAwareSplitSource(CONNECTOR_ID, TestingTransactionHandle.create(), createFixedSplitSource(20, TestingSplit::createRemoteSplit)), new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(TestingSession.testSessionBuilder().build(), CONNECTOR_ID), stage::getAllTasks), 2);
        scheduler.schedule();
        fail("expected PrestoException");
    } catch (PrestoException e) {
        assertEquals(e.getErrorCode(), NO_NODES_AVAILABLE.toErrorCode());
    }
}
Also used : NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) PrestoException(com.facebook.presto.spi.PrestoException) ThrowingNodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager) SqlStageExecution(com.facebook.presto.execution.SqlStageExecution) ConnectorAwareSplitSource(com.facebook.presto.split.ConnectorAwareSplitSource) InMemoryNodeManager(com.facebook.presto.metadata.InMemoryNodeManager) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(com.facebook.presto.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) NoOpQueryManager(com.facebook.presto.dispatcher.NoOpQueryManager) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) TestingSplit(com.facebook.presto.testing.TestingSplit) SimpleTtlNodeSelectorConfig(com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig) SubPlan(com.facebook.presto.sql.planner.SubPlan) Test(org.testng.annotations.Test)

Aggregations

SimpleTtlNodeSelectorConfig (com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig)15 NodeSelectionStats (com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats)14 NoOpQueryManager (com.facebook.presto.dispatcher.NoOpQueryManager)13 ThrowingNodeTtlFetcherManager (com.facebook.presto.ttl.nodettlfetchermanagers.ThrowingNodeTtlFetcherManager)13 NodeScheduler (com.facebook.presto.execution.scheduler.NodeScheduler)12 NodeSchedulerConfig (com.facebook.presto.execution.scheduler.NodeSchedulerConfig)12 LegacyNetworkTopology (com.facebook.presto.execution.scheduler.LegacyNetworkTopology)11 Test (org.testng.annotations.Test)11 Split (com.facebook.presto.metadata.Split)10 NodeSelector (com.facebook.presto.execution.scheduler.nodeSelection.NodeSelector)9 InternalNode (com.facebook.presto.metadata.InternalNode)9 ConnectorSplit (com.facebook.presto.spi.ConnectorSplit)9 TestingTransactionHandle (com.facebook.presto.testing.TestingTransactionHandle)9 HashSet (java.util.HashSet)9 SplitPlacementResult (com.facebook.presto.execution.scheduler.SplitPlacementResult)7 InMemoryNodeManager (com.facebook.presto.metadata.InMemoryNodeManager)7 NetworkLocationCache (com.facebook.presto.execution.scheduler.NetworkLocationCache)3 ImmutableList (com.google.common.collect.ImmutableList)3 Duration (io.airlift.units.Duration)3 Session (com.facebook.presto.Session)2