Examples with InternalNode io.prestosql.metadata.InternalNode used on opensource projects

Example 6 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class LowMemoryKillerTestingUtils method toNodeMemoryInfoList.

static List<MemoryInfo> toNodeMemoryInfoList(long maxReservedPoolBytes, long maxGeneralPoolBytes, String reservedQuery, Map<String, Map<String, Long>> queries) {
    Map<InternalNode, NodeReservation> nodeReservations = new HashMap<>();
    for (Map.Entry<String, Map<String, Long>> entry : queries.entrySet()) {
        QueryId queryId = new QueryId(entry.getKey());
        Map<String, Long> reservationByNode = entry.getValue();
        for (Map.Entry<String, Long> nodeEntry : reservationByNode.entrySet()) {
            InternalNode node = new InternalNode(nodeEntry.getKey(), URI.create("http://localhost"), new NodeVersion("version"), false);
            long bytes = nodeEntry.getValue();
            if (bytes == 0) {
                continue;
            }
            if (reservedQuery.equals(entry.getKey())) {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getReserved().add(queryId, bytes);
            } else {
                nodeReservations.computeIfAbsent(node, ignored -> new NodeReservation()).getGeneral().add(queryId, bytes);
            }
        }
    }
    ImmutableList.Builder<MemoryInfo> result = ImmutableList.builder();
    for (Map.Entry<InternalNode, NodeReservation> entry : nodeReservations.entrySet()) {
        NodeReservation nodeReservation = entry.getValue();
        ImmutableMap.Builder<MemoryPoolId, MemoryPoolInfo> pools = ImmutableMap.builder();
        if (nodeReservation.getGeneral().getTotalReservedBytes() > 0) {
            pools.put(GENERAL_POOL, new MemoryPoolInfo(maxGeneralPoolBytes, nodeReservation.getGeneral().getTotalReservedBytes(), 0, nodeReservation.getGeneral().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        if (nodeReservation.getReserved().getTotalReservedBytes() > 0) {
            pools.put(RESERVED_POOL, new MemoryPoolInfo(maxReservedPoolBytes, nodeReservation.getReserved().getTotalReservedBytes(), 0, nodeReservation.getReserved().getReservationByQuery(), ImmutableMap.of(), ImmutableMap.of()));
        }
        result.add(new MemoryInfo(7, 0.0, 0.0, new DataSize(maxReservedPoolBytes + maxGeneralPoolBytes, BYTE), pools.build()));
    }
    return result.build();
}

Example 7 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class TestNodeScheduler method testOptimizedLocalScheduling.

@Test
public void testOptimizedLocalScheduling() {
    InternalNode node1 = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node1);
    InternalNode node2 = new InternalNode("node2", URI.create("http://10.0.0.1:12"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node2);
    Set<Split> splits = new LinkedHashSet<>();
    // 20 splits with node1 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 20; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    }
    // computeAssignments just returns a mapping of nodes with splits to be assigned, it does not assign splits
    Multimap<InternalNode, Split> assignments1 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that all 20 splits are being assigned to node1 as optimized-local-scheduling is enabled
    assertEquals(assignments1.size(), 20);
    assertEquals(assignments1.keySet().size(), 2);
    assertTrue(assignments1.keySet().contains(node1));
    assertTrue(assignments1.keySet().contains(node2));
    // 19 splits with node2 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 19; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromString("10.0.0.1:12")), Lifespan.taskWide()));
    }
    Multimap<InternalNode, Split> assignments2 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that all 39 splits are being assigned (20 splits assigned to node1 and 19 splits assigned to node2)
    assertEquals(assignments2.size(), 39);
    assertEquals(assignments2.keySet().size(), 2);
    assertTrue(assignments2.keySet().contains(node1));
    assertTrue(assignments2.keySet().contains(node2));
    long node1Splits = assignments2.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
    assertEquals(node1Splits, 20);
    long node2Splits = assignments2.values().stream().map(Split::getConnectorSplit).filter(TestSplitRemote.class::isInstance).count();
    assertEquals(node2Splits, 19);
    // 1 split with node1 as local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    // 1 split with node2 as local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromString("10.0.0.1:12")), Lifespan.taskWide()));
    // splits now contains 41 splits : 21 with node1 as local node and 20 with node2 as local node
    Multimap<InternalNode, Split> assignments3 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that only 40 splits are being assigned as there is a single task
    assertEquals(assignments3.size(), 40);
    assertEquals(assignments3.keySet().size(), 2);
    assertTrue(assignments3.keySet().contains(node1));
    assertTrue(assignments3.keySet().contains(node2));
    // The first 20 splits have node1 as local, the next 19 have node2 as local, the 40th split has node1 as local and the 41st has node2 as local
    // If optimized-local-scheduling is disabled, the 41st split will be unassigned (the last slot in node2 will be taken up by the 40th split with node1 as local)
    // optimized-local-scheduling ensures that all splits that can be assigned locally will be assigned first
    node1Splits = assignments3.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
    assertEquals(node1Splits, 20);
    node2Splits = assignments3.values().stream().map(Split::getConnectorSplit).filter(TestSplitRemote.class::isInstance).count();
    assertEquals(node2Splits, 20);
}

Example 8 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class TestNodeScheduler method testEquateDistribution.

@Test
public void testEquateDistribution() {
    InternalNode node1 = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node1);
    InternalNode node2 = new InternalNode("node2", URI.create("http://10.0.0.1:12"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node2);
    InternalNode node3 = new InternalNode("node3", URI.create("http://10.0.0.1:13"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node3);
    InternalNode node4 = new InternalNode("node4", URI.create("http://10.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node4);
    Set<Split> splits = new LinkedHashSet<>();
    // 20 splits with node1 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 20; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    }
    // check that splits are divided uniformly across all nodes
    Multimap<InternalNode, Split> assignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    assertEquals(assignment.size(), 20);
    assertEquals(assignment.keySet().size(), 4);
    assertEquals(assignment.get(node1).size(), 5);
    assertEquals(assignment.get(node2).size(), 5);
    assertEquals(assignment.get(node3).size(), 5);
    assertEquals(assignment.get(node4).size(), 5);
}

Example 9 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class TestNodeScheduler method testAssignmentWhenMixedSplits.

@Test
public void testAssignmentWhenMixedSplits() {
    InternalNode node = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node);
    // Check for Split assignments till maxSplitsPerNode (20)
    Set<Split> splits = new LinkedHashSet<>();
    // 10 splits with node1 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 10; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    }
    // 10 splits with node1 as a non-local node to be assigned in the second iteration of computeAssignments
    for (int i = 0; i < 10; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    }
    // computeAssignments just returns a mapping of nodes with splits to be assigned, it does not assign splits
    Multimap<InternalNode, Split> initialAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that all splits are being assigned to node1
    assertEquals(initialAssignment.size(), 20);
    assertEquals(initialAssignment.keySet().size(), 1);
    assertTrue(initialAssignment.keySet().contains(node));
    // Check for assignment of splits beyond maxSplitsPerNode (2 splits should remain unassigned)
    // 1 split with node1 as local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    // 1 split with node1 as a non-local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    // splits now contains 22 splits : 11 with node1 as local node and 11 with node1 as a non-local node
    Multimap<InternalNode, Split> finalAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that only 20 splits are being assigned as there is a single task
    assertEquals(finalAssignment.size(), 20);
    assertEquals(finalAssignment.keySet().size(), 1);
    assertTrue(finalAssignment.keySet().contains(node));
    // When optimized-local-scheduling is enabled, all 11 splits with node1 as local node should be assigned
    long countLocalSplits = finalAssignment.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
    assertEquals(countLocalSplits, 11);
}

Example 10 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class TestNodeScheduler method testBasicAssignment.

@Test
public void testBasicAssignment() {
    setUpNodes();
    // One split for each node
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 3; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    assertEquals(assignments.entries().size(), 3);
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        assertTrue(assignments.keySet().contains(node));
    }
}