Search in sources :

Example 6 with RemoteTask

use of io.prestosql.execution.RemoteTask in project hetu-core by openlookeng.

the class TestNodeScheduler method testMaxSplitsPerNodePerTask.

@Test
public void testMaxSplitsPerNodePerTask() {
    setUpNodes();
    InternalNode newNode = new InternalNode("other4", URI.create("http://10.0.0.1:14"), NodeVersion.UNKNOWN, false);
    nodeManager.addNode(CONNECTOR_ID, newNode);
    ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
    for (int i = 0; i < 20; i++) {
        initialSplits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    }
    List<RemoteTask> tasks = new ArrayList<>();
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
        // Max out number of splits on node
        TaskId taskId = new TaskId("test", 1, 1);
        RemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId));
        nodeTaskMap.addTask(node, remoteTask);
        tasks.add(remoteTask);
    }
    TaskId taskId = new TaskId("test", 1, 2);
    RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, newNode, initialSplits.build(), nodeTaskMap.createPartitionedSplitCountTracker(newNode, taskId));
    // Max out pending splits on new node
    taskMap.put(newNode, newRemoteTask);
    nodeTaskMap.addTask(newNode, newRemoteTask);
    tasks.add(newRemoteTask);
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 5; i++) {
        splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
    // no split should be assigned to the newNode, as it already has
    // maxSplitsPerNode + maxSplitsPerNodePerTask assigned to it
    // Splits should be scheduled on the other three nodes
    assertEquals(assignments.keySet().size(), 3);
    // No splits scheduled on the maxed out node
    assertFalse(assignments.keySet().contains(newNode));
    for (RemoteTask task : tasks) {
        task.abort();
    }
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(newNode), 0);
}
Also used : TaskId(io.prestosql.execution.TaskId) ImmutableList(com.google.common.collect.ImmutableList) ArrayList(java.util.ArrayList) RemoteTask(io.prestosql.execution.RemoteTask) InternalNode(io.prestosql.metadata.InternalNode) MockSplit(io.prestosql.MockSplit) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) Split(io.prestosql.metadata.Split) TestingSplit(io.prestosql.testing.TestingSplit) MockRemoteTaskFactory(io.prestosql.execution.MockRemoteTaskFactory) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) Test(org.testng.annotations.Test)

Example 7 with RemoteTask

use of io.prestosql.execution.RemoteTask in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method testScheduleSplitsBlock.

@Test
public void testScheduleSplitsBlock() {
    StageExecutionPlan plan = createPlan(createFixedSplitSource(80, TestingSplit::createRemoteSplit));
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1);
    // schedule first 60 splits, which will cause the scheduler to block
    for (int i = 0; i <= 60; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        assertFalse(scheduleResult.isFinished());
        // blocks at 20 per node
        assertEquals(scheduleResult.getBlocked().isDone(), i != 60);
        // first three splits create new tasks
        assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
        assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
        assertPartitionedSplitCount(stage, min(i + 1, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        assertEquals(remoteTask.getPartitionedSplitCount(), 20);
    }
    // todo rewrite MockRemoteTask to fire a tate transition when splits are cleared, and then validate blocked future completes
    // drop the 20 splits from one node
    ((MockRemoteTask) stage.getAllTasks().get(0)).clearSplits();
    // schedule remaining 20 splits
    for (int i = 0; i < 20; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        // finishes when last split is fetched
        if (i == 19) {
            assertEffectivelyFinished(scheduleResult, scheduler);
        } else {
            assertFalse(scheduleResult.isFinished());
        }
        // does not block again
        assertTrue(scheduleResult.getBlocked().isDone());
        // no additional tasks will be created
        assertEquals(scheduleResult.getNewTasks().size(), 0);
        assertEquals(stage.getAllTasks().size(), 3);
        // we dropped 20 splits so start at 40 and count to 60
        assertPartitionedSplitCount(stage, min(i + 41, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        assertEquals(remoteTask.getPartitionedSplitCount(), 20);
    }
    stage.abort();
}
Also used : NodeTaskMap(io.prestosql.execution.NodeTaskMap) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) MockRemoteTask(io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask) MockRemoteTask(io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(io.prestosql.execution.RemoteTask) SqlStageExecution(io.prestosql.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Example 8 with RemoteTask

use of io.prestosql.execution.RemoteTask in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method testScheduleSplitsOneAtATime.

@Test
public void testScheduleSplitsOneAtATime() {
    StageExecutionPlan plan = createPlan(createFixedSplitSource(60, TestingSplit::createRemoteSplit));
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    try (StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1)) {
        for (int i = 0; i < 60; i++) {
            ScheduleResult scheduleResult = scheduler.schedule();
            // only finishes when last split is fetched
            if (i == 59) {
                assertEffectivelyFinished(scheduleResult, scheduler);
            } else {
                assertFalse(scheduleResult.isFinished());
            }
            // never blocks
            assertTrue(scheduleResult.getBlocked().isDone());
            // first three splits create new tasks
            assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
            assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
            assertPartitionedSplitCount(stage, min(i + 1, 60));
        }
        for (RemoteTask remoteTask : stage.getAllTasks()) {
            assertEquals(remoteTask.getPartitionedSplitCount(), 20);
        }
        stage.abort();
    }
}
Also used : NodeTaskMap(io.prestosql.execution.NodeTaskMap) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) MockRemoteTask(io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(io.prestosql.execution.RemoteTask) SqlStageExecution(io.prestosql.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Example 9 with RemoteTask

use of io.prestosql.execution.RemoteTask in project hetu-core by openlookeng.

the class TestFixedCountScheduler method testMultipleNodes.

@Test
public void testMultipleNodes() {
    FixedCountScheduler nodeScheduler = new FixedCountScheduler((node, partition, totalPartitions) -> Optional.of(taskFactory.createTableScanTask(new TaskId("test", 1, 1), node, ImmutableList.of(), new PartitionedSplitCountTracker(delta -> {
    }))), generateRandomNodes(5));
    ScheduleResult result = nodeScheduler.schedule();
    assertTrue(result.isFinished());
    assertTrue(result.getBlocked().isDone());
    assertEquals(result.getNewTasks().size(), 5);
    assertEquals(result.getNewTasks().stream().map(RemoteTask::getNodeId).collect(toImmutableSet()).size(), 5);
}
Also used : PartitionedSplitCountTracker(io.prestosql.execution.NodeTaskMap.PartitionedSplitCountTracker) IntStream(java.util.stream.IntStream) AfterClass(org.testng.annotations.AfterClass) TaskId(io.prestosql.execution.TaskId) InternalNode(io.prestosql.metadata.InternalNode) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) MockRemoteTaskFactory(io.prestosql.execution.MockRemoteTaskFactory) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Optional(java.util.Optional) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Assert.assertTrue(org.testng.Assert.assertTrue) URI(java.net.URI) NodeVersion(io.prestosql.client.NodeVersion) ExecutorService(java.util.concurrent.ExecutorService) RemoteTask(io.prestosql.execution.RemoteTask) TaskId(io.prestosql.execution.TaskId) PartitionedSplitCountTracker(io.prestosql.execution.NodeTaskMap.PartitionedSplitCountTracker) Test(org.testng.annotations.Test)

Example 10 with RemoteTask

use of io.prestosql.execution.RemoteTask in project hetu-core by openlookeng.

the class SimpleNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks, Optional<SqlStageExecution> stage) {
    Multimap<InternalNode, Split> assignment = HashMultimap.create();
    NodeMap nodeMapSlice = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMapSlice, existingTasks);
    ResettableRandomizedIterator<InternalNode> randomCandidates = randomizedNodes(nodeMapSlice, ImmutableSet.of());
    Set<InternalNode> blockedExactNodes = new HashSet<>();
    boolean splitWaitingForAnyNode = false;
    // splitsToBeRedistributed becomes true only when splits go through locality-based assignment
    boolean splitsToBeRedistributed = false;
    Set<Split> remainingSplits = new HashSet<>();
    // Check if the current stage has a TableScanNode which is reading the table for the 2nd time or beyond
    if (stage.isPresent() && stage.get().getStateMachine().getConsumerScanNode() != null) {
        try {
            // if node exists, get the TableScanNode and cast it as consumer
            TableScanNode consumer = stage.get().getStateMachine().getConsumerScanNode();
            // all tables part of this stage
            Map<PlanNodeId, TableInfo> tables = stage.get().getStageInfo().getTables();
            QualifiedObjectName tableName;
            for (Map.Entry<PlanNodeId, TableInfo> entry : tables.entrySet()) {
                tableName = entry.getValue().getTableName();
                if (tableSplitAssignmentInfo.getReuseTableScanMappingIdSplitAssignmentMap().containsKey(consumer.getReuseTableScanMappingId())) {
                    // compare splitkey using equals and then assign nodes accordingly.
                    HashMap<SplitKey, InternalNode> splitKeyNodeAssignment = tableSplitAssignmentInfo.getSplitKeyNodeAssignment(consumer.getReuseTableScanMappingId());
                    Set<SplitKey> splitKeySet = splitKeyNodeAssignment.keySet();
                    assignment.putAll(createConsumerScanNodeAssignment(tableName, splits, splitKeySet, splitKeyNodeAssignment));
                    for (Map.Entry<InternalNode, Split> nodeAssignmentEntry : assignment.entries()) {
                        InternalNode node = nodeAssignmentEntry.getKey();
                        assignmentStats.addAssignedSplit(node);
                    }
                }
            }
            log.debug("Consumer:: Assignment size is " + assignment.size() + " ,Assignment is " + assignment + " ,Assignment Stats is " + assignmentStats);
        } catch (NotImplementedException e) {
            log.error("Not a Hive Split! Other Connector Splits not supported currently. Error: " + e);
            throw new UnsupportedOperationException("Not a Hive Split! Other Connector Splits not supported currently. Error: " + e);
        }
    } else {
        // optimizedLocalScheduling enables prioritized assignment of splits to local nodes when splits contain locality information
        if (optimizedLocalScheduling) {
            // should not hit for consumer case
            for (Split split : splits) {
                if (split.isRemotelyAccessible() && !split.getAddresses().isEmpty()) {
                    List<InternalNode> candidateNodes = selectExactNodes(nodeMapSlice, split.getAddresses(), includeCoordinator);
                    Optional<InternalNode> chosenNode = candidateNodes.stream().filter(ownerNode -> assignmentStats.getTotalSplitCount(ownerNode) < maxSplitsPerNode).min(comparingInt(assignmentStats::getTotalSplitCount));
                    if (chosenNode.isPresent()) {
                        assignment.put(chosenNode.get(), split);
                        // check later
                        assignmentStats.addAssignedSplit(chosenNode.get());
                        splitsToBeRedistributed = true;
                        continue;
                    }
                }
                remainingSplits.add(split);
            }
        } else {
            remainingSplits = splits;
        }
        for (Split split : remainingSplits) {
            randomCandidates.reset();
            List<InternalNode> candidateNodes;
            if (!split.isRemotelyAccessible()) {
                candidateNodes = selectExactNodes(nodeMapSlice, split.getAddresses(), includeCoordinator);
            } else {
                candidateNodes = selectNodes(minCandidates, randomCandidates);
            }
            if (candidateNodes.isEmpty()) {
                log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMapSlice.getNodesByHost().keys());
                throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
            }
            InternalNode chosenNode = null;
            int min = Integer.MAX_VALUE;
            for (InternalNode node : candidateNodes) {
                int totalSplitCount = assignmentStats.getTotalSplitCount(node);
                if (totalSplitCount < min && totalSplitCount < maxSplitsPerNode) {
                    chosenNode = node;
                    min = totalSplitCount;
                }
            }
            if (chosenNode == null) {
                // min is guaranteed to be MAX_VALUE at this line
                for (InternalNode node : candidateNodes) {
                    int totalSplitCount = assignmentStats.getQueuedSplitCountForStage(node);
                    if (totalSplitCount < min && totalSplitCount < maxPendingSplitsPerTask) {
                        chosenNode = node;
                        min = totalSplitCount;
                    }
                }
            }
            if (chosenNode != null) {
                assignment.put(chosenNode, split);
                assignmentStats.addAssignedSplit(chosenNode);
            } else {
                if (split.isRemotelyAccessible()) {
                    splitWaitingForAnyNode = true;
                } else // Exact node set won't matter, if a split is waiting for any node
                if (!splitWaitingForAnyNode) {
                    blockedExactNodes.addAll(candidateNodes);
                }
            }
        }
    }
    ListenableFuture<?> blocked;
    if (splitWaitingForAnyNode) {
        blocked = toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    } else {
        blocked = toWhenHasSplitQueueSpaceFuture(blockedExactNodes, existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    }
    if (!stage.isPresent() || stage.get().getStateMachine().getConsumerScanNode() == null) {
        if (splitsToBeRedistributed) {
            // skip for consumer
            equateDistribution(assignment, assignmentStats, nodeMapSlice);
        }
    }
    // Check if the current stage has a TableScanNode which is reading the table for the 1st time
    if (stage.isPresent() && stage.get().getStateMachine().getProducerScanNode() != null) {
        // if node exists, get the TableScanNode and annotate it as producer
        saveProducerScanNodeAssignment(stage, assignment, assignmentStats);
    }
    // Check if its CTE node and its feeder
    if (stage.isPresent() && stage.get().getFragment().getFeederCTEId().isPresent()) {
        updateFeederNodeAndSplitCount(stage.get(), assignment);
    }
    return new SplitPlacementResult(blocked, assignment);
}
Also used : NodeScheduler.randomizedNodes(io.prestosql.execution.scheduler.NodeScheduler.randomizedNodes) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Logger(io.airlift.log.Logger) Supplier(com.google.common.base.Supplier) HashMap(java.util.HashMap) Split(io.prestosql.metadata.Split) NO_NODES_AVAILABLE(io.prestosql.spi.StandardErrorCode.NO_NODES_AVAILABLE) Multimap(com.google.common.collect.Multimap) AtomicReference(java.util.concurrent.atomic.AtomicReference) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) InetAddress(java.net.InetAddress) HashSet(java.util.HashSet) HashMultimap(com.google.common.collect.HashMultimap) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Suppliers(com.google.common.base.Suppliers) NodeTaskMap(io.prestosql.execution.NodeTaskMap) InternalNodeManager(io.prestosql.metadata.InternalNodeManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableInfo(io.prestosql.execution.TableInfo) PrestoException(io.prestosql.spi.PrestoException) Comparator.comparingInt(java.util.Comparator.comparingInt) ImmutableSet(com.google.common.collect.ImmutableSet) NotImplementedException(sun.reflect.generics.reflectiveObjects.NotImplementedException) HostAddress(io.prestosql.spi.HostAddress) Iterator(java.util.Iterator) NodeScheduler.calculateLowWatermark(io.prestosql.execution.scheduler.NodeScheduler.calculateLowWatermark) IndexedPriorityQueue(io.prestosql.execution.resourcegroups.IndexedPriorityQueue) InternalNode(io.prestosql.metadata.InternalNode) Collection(java.util.Collection) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) NodeScheduler.toWhenHasSplitQueueSpaceFuture(io.prestosql.execution.scheduler.NodeScheduler.toWhenHasSplitQueueSpaceFuture) UnknownHostException(java.net.UnknownHostException) Collectors(java.util.stream.Collectors) SetMultimap(com.google.common.collect.SetMultimap) NodeScheduler.selectNodes(io.prestosql.execution.scheduler.NodeScheduler.selectNodes) List(java.util.List) SplitKey(io.prestosql.execution.SplitKey) GENERIC_INTERNAL_ERROR(io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) Optional(java.util.Optional) NodeScheduler.selectDistributionNodes(io.prestosql.execution.scheduler.NodeScheduler.selectDistributionNodes) VisibleForTesting(com.google.common.annotations.VisibleForTesting) SqlStageExecution(io.prestosql.execution.SqlStageExecution) NodeScheduler.selectExactNodes(io.prestosql.execution.scheduler.NodeScheduler.selectExactNodes) RemoteTask(io.prestosql.execution.RemoteTask) SplitKey(io.prestosql.execution.SplitKey) NotImplementedException(sun.reflect.generics.reflectiveObjects.NotImplementedException) PrestoException(io.prestosql.spi.PrestoException) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableInfo(io.prestosql.execution.TableInfo) HashSet(java.util.HashSet) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) TableScanNode(io.prestosql.spi.plan.TableScanNode) InternalNode(io.prestosql.metadata.InternalNode) Split(io.prestosql.metadata.Split) HashMap(java.util.HashMap) Map(java.util.Map) NodeTaskMap(io.prestosql.execution.NodeTaskMap)

Aggregations

RemoteTask (io.prestosql.execution.RemoteTask)25 InternalNode (io.prestosql.metadata.InternalNode)17 Test (org.testng.annotations.Test)14 Split (io.prestosql.metadata.Split)12 ImmutableList (com.google.common.collect.ImmutableList)11 NodeTaskMap (io.prestosql.execution.NodeTaskMap)11 SqlStageExecution (io.prestosql.execution.SqlStageExecution)11 HashSet (java.util.HashSet)10 TaskId (io.prestosql.execution.TaskId)9 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)9 List (java.util.List)8 Optional (java.util.Optional)8 MockRemoteTaskFactory (io.prestosql.execution.MockRemoteTaskFactory)7 SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler (io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler)7 StageExecutionPlan (io.prestosql.sql.planner.StageExecutionPlan)7 TestingSplit (io.prestosql.testing.TestingSplit)7 HashMap (java.util.HashMap)7 Objects.requireNonNull (java.util.Objects.requireNonNull)7 ImmutableSet (com.google.common.collect.ImmutableSet)6 ImmutableSet.toImmutableSet (com.google.common.collect.ImmutableSet.toImmutableSet)6