Examples with Split io.prestosql.metadata.Split used on opensource projects

Example 11 with Split

use of io.prestosql.metadata.Split in project hetu-core by openlookeng.

the class TestNodeScheduler method testOptimizedLocalScheduling.

@Test
public void testOptimizedLocalScheduling() {
    InternalNode node1 = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node1);
    InternalNode node2 = new InternalNode("node2", URI.create("http://10.0.0.1:12"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node2);
    Set<Split> splits = new LinkedHashSet<>();
    // 20 splits with node1 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 20; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    }
    // computeAssignments just returns a mapping of nodes with splits to be assigned, it does not assign splits
    Multimap<InternalNode, Split> assignments1 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that all 20 splits are being assigned to node1 as optimized-local-scheduling is enabled
    assertEquals(assignments1.size(), 20);
    assertEquals(assignments1.keySet().size(), 2);
    assertTrue(assignments1.keySet().contains(node1));
    assertTrue(assignments1.keySet().contains(node2));
    // 19 splits with node2 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 19; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromString("10.0.0.1:12")), Lifespan.taskWide()));
    }
    Multimap<InternalNode, Split> assignments2 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that all 39 splits are being assigned (20 splits assigned to node1 and 19 splits assigned to node2)
    assertEquals(assignments2.size(), 39);
    assertEquals(assignments2.keySet().size(), 2);
    assertTrue(assignments2.keySet().contains(node1));
    assertTrue(assignments2.keySet().contains(node2));
    long node1Splits = assignments2.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
    assertEquals(node1Splits, 20);
    long node2Splits = assignments2.values().stream().map(Split::getConnectorSplit).filter(TestSplitRemote.class::isInstance).count();
    assertEquals(node2Splits, 19);
    // 1 split with node1 as local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    // 1 split with node2 as local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromString("10.0.0.1:12")), Lifespan.taskWide()));
    // splits now contains 41 splits : 21 with node1 as local node and 20 with node2 as local node
    Multimap<InternalNode, Split> assignments3 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that only 40 splits are being assigned as there is a single task
    assertEquals(assignments3.size(), 40);
    assertEquals(assignments3.keySet().size(), 2);
    assertTrue(assignments3.keySet().contains(node1));
    assertTrue(assignments3.keySet().contains(node2));
    // The first 20 splits have node1 as local, the next 19 have node2 as local, the 40th split has node1 as local and the 41st has node2 as local
    // If optimized-local-scheduling is disabled, the 41st split will be unassigned (the last slot in node2 will be taken up by the 40th split with node1 as local)
    // optimized-local-scheduling ensures that all splits that can be assigned locally will be assigned first
    node1Splits = assignments3.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
    assertEquals(node1Splits, 20);
    node2Splits = assignments3.values().stream().map(Split::getConnectorSplit).filter(TestSplitRemote.class::isInstance).count();
    assertEquals(node2Splits, 20);
}

Example 12 with Split

use of io.prestosql.metadata.Split in project hetu-core by openlookeng.

the class TestNodeScheduler method testEquateDistribution.

@Test
public void testEquateDistribution() {
    InternalNode node1 = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node1);
    InternalNode node2 = new InternalNode("node2", URI.create("http://10.0.0.1:12"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node2);
    InternalNode node3 = new InternalNode("node3", URI.create("http://10.0.0.1:13"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node3);
    InternalNode node4 = new InternalNode("node4", URI.create("http://10.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node4);
    Set<Split> splits = new LinkedHashSet<>();
    // 20 splits with node1 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 20; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    }
    // check that splits are divided uniformly across all nodes
    Multimap<InternalNode, Split> assignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    assertEquals(assignment.size(), 20);
    assertEquals(assignment.keySet().size(), 4);
    assertEquals(assignment.get(node1).size(), 5);
    assertEquals(assignment.get(node2).size(), 5);
    assertEquals(assignment.get(node3).size(), 5);
    assertEquals(assignment.get(node4).size(), 5);
}

Example 13 with Split

use of io.prestosql.metadata.Split in project hetu-core by openlookeng.

the class TestNodeScheduler method testAssignmentWhenMixedSplits.

@Test
public void testAssignmentWhenMixedSplits() {
    InternalNode node = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, node);
    // Check for Split assignments till maxSplitsPerNode (20)
    Set<Split> splits = new LinkedHashSet<>();
    // 10 splits with node1 as local node to be assigned in the first iteration of computeAssignments
    for (int i = 0; i < 10; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    }
    // 10 splits with node1 as a non-local node to be assigned in the second iteration of computeAssignments
    for (int i = 0; i < 10; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    }
    // computeAssignments just returns a mapping of nodes with splits to be assigned, it does not assign splits
    Multimap<InternalNode, Split> initialAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that all splits are being assigned to node1
    assertEquals(initialAssignment.size(), 20);
    assertEquals(initialAssignment.keySet().size(), 1);
    assertTrue(initialAssignment.keySet().contains(node));
    // Check for assignment of splits beyond maxSplitsPerNode (2 splits should remain unassigned)
    // 1 split with node1 as local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
    // 1 split with node1 as a non-local node
    splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    // splits now contains 22 splits : 11 with node1 as local node and 11 with node1 as a non-local node
    Multimap<InternalNode, Split> finalAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // Check that only 20 splits are being assigned as there is a single task
    assertEquals(finalAssignment.size(), 20);
    assertEquals(finalAssignment.keySet().size(), 1);
    assertTrue(finalAssignment.keySet().contains(node));
    // When optimized-local-scheduling is enabled, all 11 splits with node1 as local node should be assigned
    long countLocalSplits = finalAssignment.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
    assertEquals(countLocalSplits, 11);
}

Example 14 with Split

use of io.prestosql.metadata.Split in project hetu-core by openlookeng.

the class TestNodeScheduler method testBasicAssignment.

@Test
public void testBasicAssignment() {
    setUpNodes();
    // One split for each node
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 3; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    assertEquals(assignments.entries().size(), 3);
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        assertTrue(assignments.keySet().contains(node));
    }
}

Example 15 with Split

use of io.prestosql.metadata.Split in project hetu-core by openlookeng.

the class TestNodeScheduler method testAssignmentWhenNoNodes.

// Test exception throw when no nodes available to schedule
@Test
public void testAssignmentWhenNoNodes() {
    Set<Split> splits = new HashSet<>();
    splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    assertPrestoExceptionThrownBy(() -> nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty())).hasErrorCode(NO_NODES_AVAILABLE).hasMessageMatching("No nodes available to run query");
}