Examples with IntermediateResultPartitionID org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID used on opensource projects

Example 11 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class EdgeManagerBuildUtil method connectAllToAll.

private static void connectAllToAll(ExecutionVertex[] taskVertices, IntermediateResult intermediateResult) {
    List<IntermediateResultPartitionID> consumedPartitions = Arrays.stream(intermediateResult.getPartitions()).map(IntermediateResultPartition::getPartitionId).collect(Collectors.toList());
    ConsumedPartitionGroup consumedPartitionGroup = createAndRegisterConsumedPartitionGroupToEdgeManager(consumedPartitions, intermediateResult);
    for (ExecutionVertex ev : taskVertices) {
        ev.addConsumedPartitionGroup(consumedPartitionGroup);
    }
    List<ExecutionVertexID> consumerVertices = Arrays.stream(taskVertices).map(ExecutionVertex::getID).collect(Collectors.toList());
    ConsumerVertexGroup consumerVertexGroup = ConsumerVertexGroup.fromMultipleVertices(consumerVertices);
    for (IntermediateResultPartition partition : intermediateResult.getPartitions()) {
        partition.addConsumers(consumerVertexGroup);
    }
}

Example 12 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class TaskDeploymentDescriptorFactory method computeConsumedPartitionShuffleDescriptors.

private MaybeOffloaded<ShuffleDescriptor[]> computeConsumedPartitionShuffleDescriptors(ConsumedPartitionGroup consumedPartitionGroup) throws IOException {
    ShuffleDescriptor[] shuffleDescriptors = new ShuffleDescriptor[consumedPartitionGroup.size()];
    // Each edge is connected to a different result partition
    int i = 0;
    for (IntermediateResultPartitionID partitionId : consumedPartitionGroup) {
        shuffleDescriptors[i++] = getConsumedPartitionShuffleDescriptor(resultPartitionRetriever.apply(partitionId), partitionDeploymentConstraint);
    }
    return serializeAndTryOffloadShuffleDescriptors(shuffleDescriptors);
}

Example 13 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method testBuildDeploymentDescriptor.

@Test
public void testBuildDeploymentDescriptor() throws Exception {
    final JobVertexID jid1 = new JobVertexID();
    final JobVertexID jid2 = new JobVertexID();
    final JobVertexID jid3 = new JobVertexID();
    final JobVertexID jid4 = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", jid1);
    JobVertex v2 = new JobVertex("v2", jid2);
    JobVertex v3 = new JobVertex("v3", jid3);
    JobVertex v4 = new JobVertex("v4", jid4);
    v1.setParallelism(10);
    v2.setParallelism(10);
    v3.setParallelism(10);
    v4.setParallelism(10);
    v1.setInvokableClass(BatchTask.class);
    v2.setInvokableClass(BatchTask.class);
    v3.setInvokableClass(BatchTask.class);
    v4.setInvokableClass(BatchTask.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(v1, v2, v3, v4);
    final JobID jobId = jobGraph.getJobID();
    DirectScheduledExecutorService executor = new DirectScheduledExecutorService();
    DefaultExecutionGraph eg = TestingDefaultExecutionGraphBuilder.newBuilder().setJobGraph(jobGraph).setFutureExecutor(executor).setIoExecutor(executor).setBlobWriter(blobWriter).build();
    eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
    checkJobOffloaded(eg);
    ExecutionJobVertex ejv = eg.getAllVertices().get(jid2);
    ExecutionVertex vertex = ejv.getTaskVertices()[3];
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    final CompletableFuture<TaskDeploymentDescriptor> tdd = new CompletableFuture<>();
    taskManagerGateway.setSubmitConsumer(FunctionUtils.uncheckedConsumer(taskDeploymentDescriptor -> {
        taskDeploymentDescriptor.loadBigData(blobCache);
        tdd.complete(taskDeploymentDescriptor);
    }));
    final LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway).createTestingLogicalSlot();
    assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
    vertex.getCurrentExecutionAttempt().transitionState(ExecutionState.SCHEDULED);
    vertex.getCurrentExecutionAttempt().registerProducedPartitions(slot.getTaskManagerLocation(), true).get();
    vertex.deployToSlot(slot);
    assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
    checkTaskOffloaded(eg, vertex.getJobvertexId());
    TaskDeploymentDescriptor descr = tdd.get();
    assertNotNull(descr);
    JobInformation jobInformation = descr.getSerializedJobInformation().deserializeValue(getClass().getClassLoader());
    TaskInformation taskInformation = descr.getSerializedTaskInformation().deserializeValue(getClass().getClassLoader());
    assertEquals(jobId, descr.getJobId());
    assertEquals(jobId, jobInformation.getJobId());
    assertEquals(jid2, taskInformation.getJobVertexId());
    assertEquals(3, descr.getSubtaskIndex());
    assertEquals(10, taskInformation.getNumberOfSubtasks());
    assertEquals(BatchTask.class.getName(), taskInformation.getInvokableClassName());
    assertEquals("v2", taskInformation.getTaskName());
    Collection<ResultPartitionDeploymentDescriptor> producedPartitions = descr.getProducedPartitions();
    Collection<InputGateDeploymentDescriptor> consumedPartitions = descr.getInputGates();
    assertEquals(2, producedPartitions.size());
    assertEquals(1, consumedPartitions.size());
    Iterator<ResultPartitionDeploymentDescriptor> iteratorProducedPartitions = producedPartitions.iterator();
    Iterator<InputGateDeploymentDescriptor> iteratorConsumedPartitions = consumedPartitions.iterator();
    assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
    assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
    ShuffleDescriptor[] shuffleDescriptors = iteratorConsumedPartitions.next().getShuffleDescriptors();
    assertEquals(10, shuffleDescriptors.length);
    Iterator<ConsumedPartitionGroup> iteratorConsumedPartitionGroup = vertex.getAllConsumedPartitionGroups().iterator();
    int idx = 0;
    for (IntermediateResultPartitionID partitionId : iteratorConsumedPartitionGroup.next()) {
        assertEquals(partitionId, shuffleDescriptors[idx++].getResultPartitionID().getPartitionId());
    }
}

Example 14 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class ExecutionGraphPartitionReleaseTest method testStrategyNotifiedOfUnFinishedVertices.

@Test
public void testStrategyNotifiedOfUnFinishedVertices() throws Exception {
    // setup a pipeline of 2 failover regions (f1 -> f2), where
    // f1 is just a source
    // f2 consists of 3 operators (o1,o2,o3), where o1 consumes f1, and o2/o3 consume o1
    final JobVertex sourceVertex = ExecutionGraphTestUtils.createNoOpVertex("source", 1);
    final JobVertex operator1Vertex = ExecutionGraphTestUtils.createNoOpVertex("operator1", 1);
    final JobVertex operator2Vertex = ExecutionGraphTestUtils.createNoOpVertex("operator2", 1);
    final JobVertex operator3Vertex = ExecutionGraphTestUtils.createNoOpVertex("operator3", 1);
    operator1Vertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);
    operator2Vertex.connectNewDataSetAsInput(operator1Vertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    operator3Vertex.connectNewDataSetAsInput(operator1Vertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    // setup partition tracker to intercept partition release calls
    final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker();
    final Queue<ResultPartitionID> releasedPartitions = new ArrayDeque<>();
    partitionTracker.setStopTrackingAndReleasePartitionsConsumer(partitionIds -> releasedPartitions.add(partitionIds.iterator().next()));
    final SchedulerBase scheduler = createScheduler(partitionTracker, sourceVertex, operator1Vertex, operator2Vertex, operator3Vertex);
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    mainThreadExecutor.execute(() -> {
        final Execution sourceExecution = getCurrentExecution(sourceVertex, executionGraph);
        // finish the source; this should not result in any release calls since the
        // consumer o1 was not finished
        scheduler.updateTaskExecutionState(new TaskExecutionState(sourceExecution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, empty());
    });
    mainThreadExecutor.execute(() -> {
        final Execution operator1Execution = getCurrentExecution(operator1Vertex, executionGraph);
        // release calls since not all operators of the pipelined region are finished
        for (final IntermediateResultPartitionID partitionId : operator1Execution.getVertex().getProducedPartitions().keySet()) {
            scheduler.notifyPartitionDataAvailable(new ResultPartitionID(partitionId, operator1Execution.getAttemptId()));
        }
        scheduler.updateTaskExecutionState(new TaskExecutionState(operator1Execution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, empty());
    });
    mainThreadExecutor.execute(() -> {
        final Execution operator2Execution = getCurrentExecution(operator2Vertex, executionGraph);
        // finish o2; this should not result in any release calls since o3 was not
        // finished
        scheduler.updateTaskExecutionState(new TaskExecutionState(operator2Execution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, empty());
    });
    mainThreadExecutor.execute(() -> {
        final Execution operator2Execution = getCurrentExecution(operator2Vertex, executionGraph);
        // reset o2
        operator2Execution.getVertex().resetForNewExecution();
        assertThat(releasedPartitions, empty());
    });
    mainThreadExecutor.execute(() -> {
        final Execution operator3Execution = getCurrentExecution(operator3Vertex, executionGraph);
        // finish o3; this should not result in any release calls since o2 was reset
        scheduler.updateTaskExecutionState(new TaskExecutionState(operator3Execution.getAttemptId(), ExecutionState.FINISHED));
        assertThat(releasedPartitions, empty());
    });
}

Example 15 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class ExecutionGraphResultPartitionAvailabilityCheckerTest method testPartitionAvailabilityCheck.

@Test
public void testPartitionAvailabilityCheck() {
    final IntermediateResultPartitionID irp1ID = new IntermediateResultPartitionID();
    final IntermediateResultPartitionID irp2ID = new IntermediateResultPartitionID();
    final IntermediateResultPartitionID irp3ID = new IntermediateResultPartitionID();
    final IntermediateResultPartitionID irp4ID = new IntermediateResultPartitionID();
    final Map<IntermediateResultPartitionID, Boolean> expectedAvailability = new HashMap<IntermediateResultPartitionID, Boolean>() {

        {
            put(irp1ID, true);
            put(irp2ID, false);
            put(irp3ID, false);
            put(irp4ID, true);
        }
    };
    // let the partition tracker respect the expected availability result
    final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker();
    partitionTracker.setIsPartitionTrackedFunction(rpID -> expectedAvailability.get(rpID.getPartitionId()));
    // the execution attempt ID should make no difference in this case
    final Function<IntermediateResultPartitionID, ResultPartitionID> partitionIDMapper = intermediateResultPartitionID -> new ResultPartitionID(intermediateResultPartitionID, new ExecutionAttemptID());
    final ResultPartitionAvailabilityChecker resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(partitionIDMapper, partitionTracker);
    for (IntermediateResultPartitionID irpID : expectedAvailability.keySet()) {
        assertEquals(expectedAvailability.get(irpID), resultPartitionAvailabilityChecker.isAvailable(irpID));
    }
}