Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 36 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class SystemSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingContext splitSchedulingContext) {
    SystemTableLayoutHandle layoutHandle = (SystemTableLayoutHandle) layout;
    SystemTableHandle tableHandle = layoutHandle.getTable();
    TupleDomain<ColumnHandle> constraint = layoutHandle.getConstraint();
    SystemTable systemTable = tables.getSystemTable(session, tableHandle.getSchemaTableName()).orElseThrow(() -> new PrestoException(NOT_FOUND, format("Table %s not found", tableHandle.getSchemaTableName())));
    Distribution tableDistributionMode = systemTable.getDistribution();
    if (tableDistributionMode == SINGLE_COORDINATOR) {
        HostAddress address = nodeManager.getCurrentNode().getHostAndPort();
        ConnectorSplit split = new SystemSplit(tableHandle.getConnectorId(), tableHandle, address, constraint);
        return new FixedSplitSource(ImmutableList.of(split));
    }
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    ImmutableSet.Builder<InternalNode> nodes = ImmutableSet.builder();
    if (tableDistributionMode == ALL_COORDINATORS) {
        nodes.addAll(nodeManager.getCoordinators());
    } else if (tableDistributionMode == ALL_NODES) {
        nodes.addAll(nodeManager.getNodes(ACTIVE).stream().filter(node -> !node.isResourceManager()).collect(toImmutableSet()));
    }
    Set<InternalNode> nodeSet = nodes.build();
    for (InternalNode node : nodeSet) {
        splits.add(new SystemSplit(tableHandle.getConnectorId(), tableHandle, node.getHostAndPort(), constraint));
    }
    return new FixedSplitSource(splits.build());
}

Example 37 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class SystemPartitioningHandle method getNodePartitionMap.

public NodePartitionMap getNodePartitionMap(Session session, NodeScheduler nodeScheduler) {
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, null);
    List<InternalNode> nodes;
    if (partitioning == SystemPartitioning.COORDINATOR_ONLY) {
        nodes = ImmutableList.of(nodeSelector.selectCurrentNode());
    } else if (partitioning == SystemPartitioning.SINGLE) {
        nodes = nodeSelector.selectRandomNodes(1);
    } else if (partitioning == SystemPartitioning.FIXED) {
        nodes = nodeSelector.selectRandomNodes(min(getHashPartitionCount(session), getMaxTasksPerStage(session)));
    } else {
        throw new IllegalArgumentException("Unsupported plan distribution " + partitioning);
    }
    checkCondition(!nodes.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available");
    return new NodePartitionMap(nodes, split -> {
        throw new UnsupportedOperationException("System distribution does not support source splits");
    });
}

Example 38 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testOutOfOrderUpdatesIgnored.

@Test(timeOut = 15_000)
public void testOutOfOrderUpdatesIgnored() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), 2);
    assertQueryInfos(provider.getClusterQueries(), 2, 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", FINISHED), 0);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), 1);
    assertQueryInfos(provider.getClusterQueries(), 2, 1);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", FINISHED), 2);
    assertQueryInfos(provider.getClusterQueries(), 2, 2);
}

Example 39 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testResourceGroupsMerged.

@Test(timeOut = 15_000)
public void testResourceGroupsMerged() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node5", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node6", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node4"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node5"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node6"));
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "rg4", GENERAL_POOL), 0);
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node3", createQueryInfo("3", FINISHED, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node4", createQueryInfo("4", FAILED, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 1, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 0, DataSize.valueOf("1MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 1, DataSize.valueOf("2MB"));
    assertResourceGroup(provider, "node5", "rg4", 1, 1, DataSize.valueOf("2MB"));
    // Add queries which are in non-terminal states other than RUNNING and QUEUED
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", WAITING_FOR_RESOURCES, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node2", createQueryInfo("6", DISPATCHING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node3", createQueryInfo("7", PLANNING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node4", createQueryInfo("8", STARTING, "rg4", GENERAL_POOL), 0);
    provider.registerQueryHeartbeat("node5", createQueryInfo("9", FINISHING, "rg4", GENERAL_POOL), 0);
    assertResourceGroup(provider, "node1", "rg4", 0, 5, DataSize.valueOf("5MB"));
    assertResourceGroup(provider, "node2", "rg4", 1, 4, DataSize.valueOf("5MB"));
    assertResourceGroup(provider, "node3", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node4", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node5", "rg4", 1, 5, DataSize.valueOf("6MB"));
    assertResourceGroup(provider, "node6", "rg4", 1, 6, DataSize.valueOf("7MB"));
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node2").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node3").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node4").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node5").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node6").isEmpty());
}

Example 40 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testQueryInfo.

@Test(timeOut = 15_000)
public void testQueryInfo() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", RUNNING), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHING), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    // Update query 2 to FINISHED to verify this is now completed in the resource manager
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 3);
    // Mix in queries from another coordinator
    provider.registerQueryHeartbeat("node2", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 8, 5);
    // Expire completed queries
    Thread.sleep(SECONDS.toMillis(5));
    assertQueryInfos(provider.getClusterQueries(), 8, 5);
    // Expire all queries
    Thread.sleep(SECONDS.toMillis(5));
    assertQueryInfos(provider.getClusterQueries(), 0, 0);
}