Examples with LogicalSlot org.apache.flink.runtime.jobmaster.LogicalSlot used on opensource projects

Example 1 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class Execution method releaseAssignedResource.

/**
 * Releases the assigned resource and completes the release future once the assigned resource
 * has been successfully released.
 *
 * @param cause for the resource release, null if none
 */
private void releaseAssignedResource(@Nullable Throwable cause) {
    assertRunningInJobMasterMainThread();
    final LogicalSlot slot = assignedResource;
    if (slot != null) {
        ComponentMainThreadExecutor jobMasterMainThreadExecutor = getVertex().getExecutionGraphAccessor().getJobMasterMainThreadExecutor();
        slot.releaseSlot(cause).whenComplete((Object ignored, Throwable throwable) -> {
            jobMasterMainThreadExecutor.assertRunningInMainThread();
            if (throwable != null) {
                releaseFuture.completeExceptionally(throwable);
            } else {
                releaseFuture.complete(null);
            }
        });
    } else {
        // no assigned resource --> we can directly complete the release future
        releaseFuture.complete(null);
    }
}

Example 2 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class Execution method sendUpdatePartitionInfoRpcCall.

/**
 * Update the partition infos on the assigned resource.
 *
 * @param partitionInfos for the remote task
 */
private void sendUpdatePartitionInfoRpcCall(final Iterable<PartitionInfo> partitionInfos) {
    final LogicalSlot slot = assignedResource;
    if (slot != null) {
        final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
        final TaskManagerLocation taskManagerLocation = slot.getTaskManagerLocation();
        CompletableFuture<Acknowledge> updatePartitionsResultFuture = taskManagerGateway.updatePartitions(attemptId, partitionInfos, rpcTimeout);
        updatePartitionsResultFuture.whenCompleteAsync((ack, failure) -> {
            // fail if there was a failure
            if (failure != null) {
                fail(new IllegalStateException("Update to task [" + getVertexWithAttempt() + "] on TaskManager " + taskManagerLocation + " failed", failure));
            }
        }, getVertex().getExecutionGraphAccessor().getJobMasterMainThreadExecutor());
    }
}

Example 3 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class CheckpointCoordinatorTest method testTasksFinishDuringTriggering.

@Test
public void testTasksFinishDuringTriggering() throws Exception {
    JobVertexID jobVertexID1 = new JobVertexID();
    JobVertexID jobVertexID2 = new JobVertexID();
    ExecutionGraph graph = new CheckpointCoordinatorTestingUtils.CheckpointExecutionGraphBuilder().setTransitToRunning(false).addJobVertex(jobVertexID1, 1, 256).addJobVertex(jobVertexID2, 1, 256).build();
    ExecutionJobVertex jobVertex1 = graph.getJobVertex(jobVertexID1);
    ExecutionVertex taskVertex = jobVertex1.getTaskVertices()[0];
    ExecutionJobVertex jobVertex2 = graph.getJobVertex(jobVertexID2);
    ExecutionVertex taskVertex2 = jobVertex2.getTaskVertices()[0];
    AtomicBoolean checkpointAborted = new AtomicBoolean(false);
    LogicalSlot slot1 = new TestingLogicalSlotBuilder().setTaskManagerGateway(new SimpleAckingTaskManagerGateway() {

        @Override
        public CompletableFuture<Acknowledge> triggerCheckpoint(ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long timestamp, CheckpointOptions checkpointOptions) {
            taskVertex.getCurrentExecutionAttempt().markFinished();
            return FutureUtils.completedExceptionally(new RpcException(""));
        }
    }).createTestingLogicalSlot();
    LogicalSlot slot2 = new TestingLogicalSlotBuilder().setTaskManagerGateway(new SimpleAckingTaskManagerGateway() {

        @Override
        public void notifyCheckpointAborted(ExecutionAttemptID executionAttemptID, JobID jobId, long checkpointId, long latestCompletedCheckpointId, long timestamp) {
            checkpointAborted.set(true);
        }
    }).createTestingLogicalSlot();
    ExecutionGraphTestUtils.setVertexResource(taskVertex, slot1);
    taskVertex.getCurrentExecutionAttempt().transitionState(ExecutionState.RUNNING);
    ExecutionGraphTestUtils.setVertexResource(taskVertex2, slot2);
    taskVertex2.getCurrentExecutionAttempt().transitionState(ExecutionState.RUNNING);
    CheckpointCoordinator checkpointCoordinator = new CheckpointCoordinatorBuilder().setExecutionGraph(graph).setTimer(manuallyTriggeredScheduledExecutor).setAllowCheckpointsAfterTasksFinished(true).build();
    // nothing should be happening
    assertEquals(0, checkpointCoordinator.getNumberOfPendingCheckpoints());
    assertEquals(0, checkpointCoordinator.getNumberOfRetainedSuccessfulCheckpoints());
    // trigger the first checkpoint. this will not fail because we allow checkpointing even with
    // finished tasks
    final CompletableFuture<CompletedCheckpoint> checkpointFuture = checkpointCoordinator.triggerCheckpoint(false);
    manuallyTriggeredScheduledExecutor.triggerAll();
    assertTrue(checkpointFuture.isCompletedExceptionally());
    assertTrue(checkpointAborted.get());
}

Example 4 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method testBuildDeploymentDescriptor.

@Test
public void testBuildDeploymentDescriptor() throws Exception {
    final JobVertexID jid1 = new JobVertexID();
    final JobVertexID jid2 = new JobVertexID();
    final JobVertexID jid3 = new JobVertexID();
    final JobVertexID jid4 = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", jid1);
    JobVertex v2 = new JobVertex("v2", jid2);
    JobVertex v3 = new JobVertex("v3", jid3);
    JobVertex v4 = new JobVertex("v4", jid4);
    v1.setParallelism(10);
    v2.setParallelism(10);
    v3.setParallelism(10);
    v4.setParallelism(10);
    v1.setInvokableClass(BatchTask.class);
    v2.setInvokableClass(BatchTask.class);
    v3.setInvokableClass(BatchTask.class);
    v4.setInvokableClass(BatchTask.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
    final JobGraph jobGraph = JobGraphTestUtils.batchJobGraph(v1, v2, v3, v4);
    final JobID jobId = jobGraph.getJobID();
    DirectScheduledExecutorService executor = new DirectScheduledExecutorService();
    DefaultExecutionGraph eg = TestingDefaultExecutionGraphBuilder.newBuilder().setJobGraph(jobGraph).setFutureExecutor(executor).setIoExecutor(executor).setBlobWriter(blobWriter).build();
    eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
    checkJobOffloaded(eg);
    ExecutionJobVertex ejv = eg.getAllVertices().get(jid2);
    ExecutionVertex vertex = ejv.getTaskVertices()[3];
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    final CompletableFuture<TaskDeploymentDescriptor> tdd = new CompletableFuture<>();
    taskManagerGateway.setSubmitConsumer(FunctionUtils.uncheckedConsumer(taskDeploymentDescriptor -> {
        taskDeploymentDescriptor.loadBigData(blobCache);
        tdd.complete(taskDeploymentDescriptor);
    }));
    final LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway).createTestingLogicalSlot();
    assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
    vertex.getCurrentExecutionAttempt().transitionState(ExecutionState.SCHEDULED);
    vertex.getCurrentExecutionAttempt().registerProducedPartitions(slot.getTaskManagerLocation(), true).get();
    vertex.deployToSlot(slot);
    assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
    checkTaskOffloaded(eg, vertex.getJobvertexId());
    TaskDeploymentDescriptor descr = tdd.get();
    assertNotNull(descr);
    JobInformation jobInformation = descr.getSerializedJobInformation().deserializeValue(getClass().getClassLoader());
    TaskInformation taskInformation = descr.getSerializedTaskInformation().deserializeValue(getClass().getClassLoader());
    assertEquals(jobId, descr.getJobId());
    assertEquals(jobId, jobInformation.getJobId());
    assertEquals(jid2, taskInformation.getJobVertexId());
    assertEquals(3, descr.getSubtaskIndex());
    assertEquals(10, taskInformation.getNumberOfSubtasks());
    assertEquals(BatchTask.class.getName(), taskInformation.getInvokableClassName());
    assertEquals("v2", taskInformation.getTaskName());
    Collection<ResultPartitionDeploymentDescriptor> producedPartitions = descr.getProducedPartitions();
    Collection<InputGateDeploymentDescriptor> consumedPartitions = descr.getInputGates();
    assertEquals(2, producedPartitions.size());
    assertEquals(1, consumedPartitions.size());
    Iterator<ResultPartitionDeploymentDescriptor> iteratorProducedPartitions = producedPartitions.iterator();
    Iterator<InputGateDeploymentDescriptor> iteratorConsumedPartitions = consumedPartitions.iterator();
    assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
    assertEquals(10, iteratorProducedPartitions.next().getNumberOfSubpartitions());
    ShuffleDescriptor[] shuffleDescriptors = iteratorConsumedPartitions.next().getShuffleDescriptors();
    assertEquals(10, shuffleDescriptors.length);
    Iterator<ConsumedPartitionGroup> iteratorConsumedPartitionGroup = vertex.getAllConsumedPartitionGroups().iterator();
    int idx = 0;
    for (IntermediateResultPartitionID partitionId : iteratorConsumedPartitionGroup.next()) {
        assertEquals(partitionId, shuffleDescriptors[idx++].getResultPartitionID().getPartitionId());
    }
}

Example 5 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class SharedSlot method allocateLogicalSlot.

/**
 * Registers an allocation request for a logical slot.
 *
 * @return the logical slot
 */
public LogicalSlot allocateLogicalSlot() {
    LOG.debug("Allocating logical slot from shared slot ({})", physicalSlotRequestId);
    Preconditions.checkState(state == State.ALLOCATED, "The shared slot has already been released.");
    final LogicalSlot slot = new SingleLogicalSlot(new SlotRequestId(), physicalSlot, Locality.UNKNOWN, this, slotWillBeOccupiedIndefinitely);
    allocatedLogicalSlots.put(slot.getSlotRequestId(), slot);
    return slot;
}