Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 31 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestRandomResourceManagerAddressSelector method testAddressSelectionNoContext.

@Test
public void testAddressSelectionNoContext() {
    InMemoryNodeManager internalNodeManager = new InMemoryNodeManager();
    RandomResourceManagerAddressSelector selector = new RandomResourceManagerAddressSelector(internalNodeManager, hostAndPorts -> Optional.of(hostAndPorts.get(0)));
    internalNodeManager.addNode(CONNECTOR_ID, new InternalNode("1", URI.create("local://localhost:123/1"), OptionalInt.empty(), "1", false, true));
    internalNodeManager.addNode(CONNECTOR_ID, new InternalNode("2", URI.create("local://localhost:456/1"), OptionalInt.of(2), "1", false, true));
    internalNodeManager.addNode(CONNECTOR_ID, new InternalNode("3", URI.create("local://localhost:789/2"), OptionalInt.of(3), "1", false, true));
    Optional<SimpleAddressSelector.SimpleAddress> address = selector.selectAddress(Optional.empty());
    assertTrue(address.isPresent());
    assertEquals(address.get().getHostAndPort(), HostAndPort.fromParts("localhost", 2));
}

Example 32 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testNonLeafResourceGroupsMerged.

@Test
public void testNonLeafResourceGroupsMerged() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node5", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node6", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node4"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node5"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node6"));
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    long query5Sequence = 0;
    long query6Sequence = 0;
    long query7Sequence = 0;
    long query8Sequence = 0;
    long query9Sequence = 0;
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "root.rg4", GENERAL_POOL), query1Sequence++);
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 0);
    provider.registerQueryHeartbeat("node2", createQueryInfo("2", RUNNING, "root.rg4", GENERAL_POOL), query2Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 1, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    provider.registerQueryHeartbeat("node3", createQueryInfo("3", FINISHED, "root.rg4", GENERAL_POOL), query3Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 1, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 0);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node3", "root", 0, 0, 1, 1);
    assertResourceGroup(provider, "node4", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node4", "root", 0, 0, 1, 1);
    provider.registerQueryHeartbeat("node4", createQueryInfo("4", FAILED, "root.rg4", GENERAL_POOL), query4Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 1, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 1);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 0, DataSize.valueOf("1MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 0);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node3", "root", 0, 0, 1, 1);
    assertResourceGroup(provider, "node4", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node4", "root", 0, 0, 1, 1);
    assertResourceGroup(provider, "node5", "root.rg4", 1, 1, DataSize.valueOf("2MB"));
    assertNonLeafResourceGroup(provider, "node5", "root", 0, 0, 1, 1);
    // Add queries which are in non-terminal states other than RUNNING and QUEUED
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", WAITING_FOR_RESOURCES, "root.rg4", GENERAL_POOL), query5Sequence++);
    provider.registerQueryHeartbeat("node2", createQueryInfo("6", DISPATCHING, "root.rg4", GENERAL_POOL), query6Sequence++);
    provider.registerQueryHeartbeat("node3", createQueryInfo("7", PLANNING, "root.rg4", GENERAL_POOL), query7Sequence++);
    provider.registerQueryHeartbeat("node4", createQueryInfo("8", STARTING, "root.rg4", GENERAL_POOL), query8Sequence++);
    provider.registerQueryHeartbeat("node5", createQueryInfo("9", FINISHING, "root.rg4", GENERAL_POOL), query9Sequence++);
    assertResourceGroup(provider, "node1", "root.rg4", 0, 5, DataSize.valueOf("5MB"));
    assertNonLeafResourceGroup(provider, "node1", "root", 0, 0, 0, 5);
    assertResourceGroup(provider, "node2", "root.rg4", 1, 4, DataSize.valueOf("5MB"));
    assertNonLeafResourceGroup(provider, "node2", "root", 0, 0, 1, 4);
    assertResourceGroup(provider, "node3", "root.rg4", 1, 5, DataSize.valueOf("6MB"));
    assertNonLeafResourceGroup(provider, "node3", "root", 0, 0, 1, 5);
    assertResourceGroup(provider, "node4", "root.rg4", 1, 5, DataSize.valueOf("6MB"));
    assertNonLeafResourceGroup(provider, "node4", "root", 0, 0, 1, 5);
    assertResourceGroup(provider, "node5", "root.rg4", 1, 5, DataSize.valueOf("6MB"));
    assertNonLeafResourceGroup(provider, "node5", "root", 0, 0, 1, 5);
    assertResourceGroup(provider, "node6", "root.rg4", 1, 6, DataSize.valueOf("7MB"));
    assertNonLeafResourceGroup(provider, "node6", "root", 0, 0, 1, 6);
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    nodeManager.refreshNodes();
    assertTrue(provider.getClusterResourceGroups("node1").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node2").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node3").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node4").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node5").isEmpty());
    assertTrue(provider.getClusterResourceGroups("node6").isEmpty());
}

Example 33 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testShuttingDownCoordinatorHeartbeat.

@Test(timeOut = 15_000)
public void testShuttingDownCoordinatorHeartbeat() {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addShuttingDownNode(new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("5s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHED), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FAILED), query4Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", RUNNING), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHING), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 2);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", FINISHED), query2Sequence++);
    assertQueryInfos(provider.getClusterQueries(), 4, 3);
}

Example 34 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testClusterMemoryPoolInfo.

@Test(timeOut = 15_000)
public void testClusterMemoryPoolInfo() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("nodeId4", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("4s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    // Memory pool starts off empty
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 0, 0, 0, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId", GENERAL_POOL, createMemoryPoolInfo(100, 2, 1)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 100, 2, 1, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Register a query and heartbeat that to the resource manager
    provider.registerQueryHeartbeat("nodeId1", createQueryInfo("1", QUEUED, "rg4", GENERAL_POOL), query1Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 0, 100, 2, 1, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create another node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId2", GENERAL_POOL, createMemoryPoolInfo(1000, 20, 10)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 0, 1100, 22, 11, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a blocked node and heartbeat to the resource manager
    provider.registerNodeHeartbeat(createNodeStatus("nodeId3", GENERAL_POOL, createMemoryPoolInfo(1, 2, 3)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 1, 1101, 24, 14, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
    // Create a node that has only reserved pool allocations
    provider.registerNodeHeartbeat(createNodeStatus("nodeId4", RESERVED_POOL, createMemoryPoolInfo(5, 3, 2)));
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 1, 1, 1101, 24, 14, Optional.of("1"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 5, 3, 2, Optional.empty());
    // Add a larger query and verify that the largest query is updated
    provider.registerQueryHeartbeat("nodeId2", createQueryInfo("2", RUNNING, "rg4", GENERAL_POOL, DataSize.valueOf("25GB")), query2Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 2, 1, 1101, 24, 14, Optional.of("2"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 5, 3, 2, Optional.empty());
    // Adding a larger reserved pool query does not affect largest query in general pool
    provider.registerQueryHeartbeat("nodeId1", createQueryInfo("3", RUNNING, "rg4", RESERVED_POOL, DataSize.valueOf("50GB")), query3Sequence++);
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 2, 1, 1101, 24, 14, Optional.of("2"));
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 1, 0, 5, 3, 2, Optional.empty());
    // Expire nodes
    Thread.sleep(SECONDS.toMillis(5));
    // All nodes expired, memory pools emptied
    assertMemoryPoolMap(provider, 2, GENERAL_POOL, 0, 0, 0, 0, 0, Optional.empty());
    assertMemoryPoolMap(provider, 2, RESERVED_POOL, 0, 0, 0, 0, 0, Optional.empty());
}

Example 35 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestResourceManagerClusterStateProvider method testResourceGroups.

@Test(timeOut = 15_000)
public void testResourceGroups() throws Exception {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node1", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node2", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    nodeManager.addNode(new ConnectorId("x"), new InternalNode("node3", URI.create("local://127.0.0.1"), NodeVersion.UNKNOWN, true));
    ResourceManagerClusterStateProvider provider = new ResourceManagerClusterStateProvider(nodeManager, new SessionPropertyManager(), 10, Duration.valueOf("4s"), Duration.valueOf("8s"), Duration.valueOf("50s"), Duration.valueOf("0s"), true, newSingleThreadScheduledExecutor());
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("local"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node1"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node2"));
    provider.registerNodeHeartbeat(createCoordinatorNodeStatus("node3"));
    assertEquals(provider.getClusterQueries(), ImmutableList.of());
    long query1Sequence = 0;
    long query2Sequence = 0;
    long query3Sequence = 0;
    long query4Sequence = 0;
    long query5Sequence = 0;
    long query6Sequence = 0;
    provider.registerQueryHeartbeat("node1", createQueryInfo("1", QUEUED, "rg1", GENERAL_POOL), query1Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("2", RUNNING, "rg2", GENERAL_POOL), query2Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("3", FINISHING, "rg3", GENERAL_POOL), query3Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("4", FINISHED, "rg4", GENERAL_POOL), query4Sequence++);
    provider.registerQueryHeartbeat("node1", createQueryInfo("5", FAILED, "rg5", GENERAL_POOL), query5Sequence++);
    assertResourceGroups(provider, "node1", 0);
    assertResourceGroups(provider, "node2", 3);
    // Add an existing leaf node from another node
    provider.registerQueryHeartbeat("node3", createQueryInfo("6", QUEUED, "rg6", GENERAL_POOL), query6Sequence++);
    assertResourceGroups(provider, "node1", 1);
    assertResourceGroups(provider, "node2", 4);
    assertResourceGroups(provider, "node3", 3);
    // Expire running queries
    Thread.sleep(SECONDS.toMillis(5));
    assertResourceGroups(provider, "node1", 0);
    assertResourceGroups(provider, "node2", 0);
    assertResourceGroups(provider, "node3", 0);
}