Examples with PlanNodeId com.facebook.presto.spi.plan.PlanNodeId used on opensource projects

Example 76 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestNodeScheduler method testCpuUsage.

@Test
public void testCpuUsage() {
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0);
    TaskId taskId1 = new TaskId("test", 1, 0, 1);
    List<Split> splits = ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()), new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, chosenNode, splits, nodeTaskMap.createTaskStatsTracker(chosenNode, taskId1));
    TaskId taskId2 = new TaskId("test", 1, 0, 2);
    RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask(taskId2, chosenNode, ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createTaskStatsTracker(chosenNode, taskId2));
    nodeTaskMap.addTask(chosenNode, remoteTask1);
    nodeTaskMap.addTask(chosenNode, remoteTask2);
    remoteTask2.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), splits).build());
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 100D);
    remoteTask1.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), splits).build());
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 200D);
    remoteTask1.abort();
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 100D);
    remoteTask2.abort();
    assertEquals(nodeTaskMap.getNodeCpuUtilizationPercentage(chosenNode), 0D);
}

Example 77 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestNodeScheduler method testMoreSplitsAssignedWhenSplitsWeightsAreSmall.

@Test
public void testMoreSplitsAssignedWhenSplitsWeightsAreSmall() {
    int standardSplitsPerNode = nodeSchedulerConfig.getMaxSplitsPerNode();
    int standardPendingSplitsPerTask = nodeSchedulerConfig.getMaxPendingSplitsPerTask();
    int fullyLoadedStandardSplitCount = standardSplitsPerNode + standardPendingSplitsPerTask;
    long weightLimitPerNode = SplitWeight.rawValueForStandardSplitCount(standardSplitsPerNode);
    long weightLimitPendingPerTask = SplitWeight.rawValueForStandardSplitCount(standardPendingSplitsPerTask);
    long fullyLoadedStandardSplitWeight = weightLimitPerNode + weightLimitPendingPerTask;
    // Single worker node
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 1);
    InternalNode workerNode = nodeSelector.selectRandomNodes(1).get(0);
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    TaskId taskId = new TaskId("test", 1, 0, 1);
    MockRemoteTaskFactory.MockRemoteTask task = remoteTaskFactory.createTableScanTask(taskId, workerNode, ImmutableList.of(), nodeTaskMap.createTaskStatsTracker(workerNode, taskId));
    TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
    ImmutableSet.Builder<Split> splitsBuilder = ImmutableSet.builderWithExpectedSize(fullyLoadedStandardSplitCount * 2);
    // Create 2x more splits than the standard split count limit, at 1/2 the standard weight
    SplitWeight halfWeight = SplitWeight.fromProportion(0.5);
    for (int i = 0; i < fullyLoadedStandardSplitCount * 2; i++) {
        splitsBuilder.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(halfWeight)));
    }
    Set<Split> splits = splitsBuilder.build();
    // Verify we arrived at the exact weight limit
    assertEquals(SplitWeight.rawValueSum(splits, Split::getSplitWeight), fullyLoadedStandardSplitWeight);
    // Node assignment limit met
    SplitPlacementResult result = nodeSelector.computeAssignments(splits, ImmutableList.of(task));
    assertEquals(result.getAssignments().get(workerNode).size(), standardSplitsPerNode * 2);
    assertEquals(SplitWeight.rawValueSum(result.getAssignments().get(workerNode), Split::getSplitWeight), weightLimitPerNode);
    // Mark all splits as running
    task.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), result.getAssignments().get(workerNode)).build());
    task.startSplits(result.getAssignments().get(workerNode).size());
    // Per task pending splits limit met
    Set<Split> remainingSplits = Sets.difference(splits, ImmutableSet.copyOf(result.getAssignments().get(workerNode)));
    SplitPlacementResult secondResults = nodeSelector.computeAssignments(remainingSplits, ImmutableList.of(task));
    assertEquals(secondResults.getAssignments().get(workerNode).size(), standardPendingSplitsPerTask * 2);
    assertEquals(SplitWeight.rawValueSum(secondResults.getAssignments().get(workerNode), Split::getSplitWeight), weightLimitPendingPerTask);
    task.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), secondResults.getAssignments().get(workerNode)).build());
    assertEquals(nodeTaskMap.getPartitionedSplitsOnNode(workerNode), // 2x fully loaded standard count, full weight limit reached
    PartitionedSplitsInfo.forSplitCountAndWeightSum(fullyLoadedStandardSplitCount * 2, fullyLoadedStandardSplitWeight));
    // No more splits assigned when full
    SplitPlacementResult resultWhenFull = nodeSelector.computeAssignments(ImmutableSet.of(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote())), ImmutableList.of(task));
    assertTrue(resultWhenFull.getAssignments().isEmpty());
}

Example 78 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestDetermineJoinDistributionType method testChoosesLeftWhenCriteriaEmpty.

@Test
public void testChoosesLeftWhenCriteriaEmpty() {
    int aRows = 1_000__00;
    int bRows = 1_0;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "1PB").overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("B1", 0)), values(ImmutableMap.of("A1", 0))));
}

Example 79 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestDetermineJoinDistributionType method testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown.

@Test
public void testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown() {
    int aRows = 10;
    int bRows = 1_000_000;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), VariableStatsEstimate.unknown())).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), VariableStatsEstimate.unknown())).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}

Example 80 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestDetermineJoinDistributionType method testReplicateLeftOuterJoin.

@Test
public void testReplicateLeftOuterJoin() {
    int aRows = 10_000;
    int bRows = 10;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(LEFT, p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.variable("A1", BIGINT), p.variable("B1", BIGINT))), ImmutableList.of(p.variable("A1", BIGINT), p.variable("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}