Examples with InMemoryNodeManager io.trino.metadata.InMemoryNodeManager used on opensource projects

Example 6 with InMemoryNodeManager

use of io.trino.metadata.InMemoryNodeManager in project trino by trinodb.

the class TestFullNodeCapableNodeAllocator method testAllocateFullOpportunistic.

@Test(timeOut = TEST_TIMEOUT)
public void testAllocateFullOpportunistic() throws Exception {
    InMemoryNodeManager nodeManager = testingNodeManager(basicNodesMap(NODE_1, NODE_2));
    setupNodeAllocatorService(nodeManager, 2);
    try (NodeAllocator nodeAllocator = nodeAllocatorService.getNodeAllocator(Q1_SESSION)) {
        // allocate both nodes as shared
        NodeAllocator.NodeLease shared1 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertAcquired(shared1);
        NodeAllocator.NodeLease shared2 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertAcquired(shared2);
        // try to allocate 2 full nodes - will block as both nodes in cluster are used
        NodeAllocator.NodeLease full1 = nodeAllocator.acquire(FULL_NODE_REQUIREMENTS);
        assertNotAcquired(full1);
        NodeAllocator.NodeLease full2 = nodeAllocator.acquire(FULL_NODE_REQUIREMENTS);
        assertNotAcquired(full2);
        // add new node to the cluster
        addNode(nodeManager, NODE_3);
        // TODO: make FullNodeCapableNodeAllocatorService react on new node added automatically
        nodeAllocatorService.wakeupProcessPendingAcquires();
        // one of the full1/full2 should be not blocked now
        assertEventually(() -> assertTrue(full1.getNode().isDone() ^ full2.getNode().isDone(), "exactly one of full1/full2 should be unblocked"));
        NodeAllocator.NodeLease fullBlocked = full1.getNode().isDone() ? full2 : full1;
        NodeAllocator.NodeLease fullNotBlocked = full1.getNode().isDone() ? full1 : full2;
        // and when unblocked one releases node the other should grab it
        fullNotBlocked.release();
        nodeAllocatorService.wakeupProcessPendingAcquires();
        assertEventually(() -> assertAcquired(fullBlocked));
    }
}

Example 7 with InMemoryNodeManager

use of io.trino.metadata.InMemoryNodeManager in project trino by trinodb.

the class TestFullNodeCapableNodeAllocator method testAllocateFullWithQueryLimitAndCatalogRequirements.

@Test(timeOut = TEST_TIMEOUT)
public void testAllocateFullWithQueryLimitAndCatalogRequirements() throws Exception {
    InMemoryNodeManager nodeManager = testingNodeManager(nodesMapBuilder().put(NODE_1, ImmutableList.of(CATALOG_1)).put(NODE_2, ImmutableList.of(CATALOG_1)).put(NODE_3, ImmutableList.of(CATALOG_2)).put(NODE_4, ImmutableList.of(CATALOG_2)).buildOrThrow());
    setupNodeAllocatorService(nodeManager, 2);
    try (NodeAllocator nodeAllocator = nodeAllocatorService.getNodeAllocator(Q1_SESSION)) {
        // allocate 2 full nodes for Q1 should not block
        NodeAllocator.NodeLease acquire1 = nodeAllocator.acquire(FULL_NODE_CATALOG_1_REQUIREMENTS);
        assertAcquired(acquire1);
        NodeAllocator.NodeLease acquire2 = nodeAllocator.acquire(FULL_NODE_CATALOG_2_REQUIREMENTS);
        assertAcquired(acquire2);
        // another allocation for CATALOG_1 will block (per query limit is 2)
        NodeAllocator.NodeLease acquire3 = nodeAllocator.acquire(FULL_NODE_CATALOG_1_REQUIREMENTS);
        assertNotAcquired(acquire3);
        // releasing CATALOG_2 node for query will unblock pending lease for CATALOG_1
        acquire2.release();
        assertEventually(() -> assertAcquired(acquire3));
    }
}

Example 8 with InMemoryNodeManager

use of io.trino.metadata.InMemoryNodeManager in project trino by trinodb.

the class TestFullNodeCapableNodeAllocator method testAllocateSharedSimple.

@Test(timeOut = TEST_TIMEOUT)
public void testAllocateSharedSimple() throws Exception {
    InMemoryNodeManager nodeManager = testingNodeManager(basicNodesMap(NODE_1, NODE_2));
    setupNodeAllocatorService(nodeManager, 1);
    try (NodeAllocator nodeAllocator = nodeAllocatorService.getNodeAllocator(Q1_SESSION)) {
        // first two allocation should not block
        NodeAllocator.NodeLease acquire1 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertAcquired(acquire1);
        NodeAllocator.NodeLease acquire2 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertAcquired(acquire2);
        // and different nodes should be assigned for each
        assertThat(Set.of(acquire1.getNode().get().getNode(), acquire2.getNode().get().getNode())).containsExactlyInAnyOrder(NODE_1, NODE_2);
        // same for subsequent two allocation (each task requires 32GB and we have 2 nodes with 64GB each)
        NodeAllocator.NodeLease acquire3 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertAcquired(acquire3);
        NodeAllocator.NodeLease acquire4 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertAcquired(acquire4);
        assertThat(Set.of(acquire3.getNode().get().getNode(), acquire4.getNode().get().getNode())).containsExactlyInAnyOrder(NODE_1, NODE_2);
        // 5th allocation should block
        NodeAllocator.NodeLease acquire5 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertNotAcquired(acquire5);
        // release acquire2 which uses
        acquire2.release();
        assertEventually(() -> {
            // we need to wait as pending acquires are processed asynchronously
            assertAcquired(acquire5);
            assertEquals(acquire5.getNode().get().getNode(), acquire2.getNode().get().getNode());
        });
        // try to acquire one more node (should block)
        NodeAllocator.NodeLease acquire6 = nodeAllocator.acquire(NO_REQUIREMENTS);
        assertNotAcquired(acquire6);
        // add new node
        addNode(nodeManager, NODE_3);
        // TODO: make FullNodeCapableNodeAllocatorService react on new node added automatically
        nodeAllocatorService.wakeupProcessPendingAcquires();
        // new node should be assigned
        assertEventually(() -> {
            assertAcquired(acquire6);
            assertEquals(acquire6.getNode().get().getNode(), NODE_3);
        });
    }
}

Example 9 with InMemoryNodeManager

use of io.trino.metadata.InMemoryNodeManager in project trino by trinodb.

the class TestFullNodeCapableNodeAllocator method testRemoveAcquiredFullNode.

@Test(timeOut = TEST_TIMEOUT)
public void testRemoveAcquiredFullNode() throws Exception {
    InMemoryNodeManager nodeManager = testingNodeManager(basicNodesMap(NODE_1));
    setupNodeAllocatorService(nodeManager, 1);
    try (NodeAllocator nodeAllocator = nodeAllocatorService.getNodeAllocator(Q1_SESSION)) {
        NodeAllocator.NodeLease acquire1 = nodeAllocator.acquire(FULL_NODE_REQUIREMENTS);
        assertAcquired(acquire1, NODE_1);
        // remove acquired node
        nodeManager.removeNode(NODE_1);
        // we should still be able to release lease for removed node
        acquire1.release();
    }
}

Example 10 with InMemoryNodeManager

use of io.trino.metadata.InMemoryNodeManager in project trino by trinodb.

the class TestFullNodeCapableNodeAllocator method testingNodeManager.

private InMemoryNodeManager testingNodeManager(Map<InternalNode, List<CatalogName>> nodeMap) {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    for (Map.Entry<InternalNode, List<CatalogName>> entry : nodeMap.entrySet()) {
        InternalNode node = entry.getKey();
        List<CatalogName> catalogs = entry.getValue();
        for (CatalogName catalog : catalogs) {
            nodeManager.addNode(catalog, node);
        }
    }
    return nodeManager;
}