Examples with LogicalSlot org.apache.flink.runtime.jobmaster.LogicalSlot used on opensource projects

Example 31 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class Execution method notifyCheckpointOnComplete.

/**
 * Notify the task of this execution about a completed checkpoint and the last subsumed
 * checkpoint id if possible.
 *
 * @param completedCheckpointId of the completed checkpoint
 * @param completedTimestamp of the completed checkpoint
 * @param lastSubsumedCheckpointId of the last subsumed checkpoint, a value of {@link
 *     org.apache.flink.runtime.checkpoint.CheckpointStoreUtil#INVALID_CHECKPOINT_ID} means no
 *     checkpoint has been subsumed.
 */
public void notifyCheckpointOnComplete(long completedCheckpointId, long completedTimestamp, long lastSubsumedCheckpointId) {
    final LogicalSlot slot = assignedResource;
    if (slot != null) {
        final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway();
        taskManagerGateway.notifyCheckpointOnComplete(attemptId, getVertex().getJobId(), completedCheckpointId, completedTimestamp, lastSubsumedCheckpointId);
    } else {
        LOG.debug("The execution has no slot assigned. This indicates that the execution is " + "no longer running.");
    }
}

Example 32 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class SharedSlot method release.

@Override
public void release(Throwable cause) {
    LOG.debug("Release shared slot ({})", physicalSlotRequestId);
    Preconditions.checkState(state == State.ALLOCATED, "The shared slot has already been released.");
    // ensures that we won't call the external release callback if there are still logical slots
    // to release
    state = State.RELEASING;
    // copy the logical slot collection to avoid ConcurrentModificationException
    // if logical slot releases cause cancellation of other executions
    // which will try to call returnLogicalSlot and modify allocatedLogicalSlots collection
    final List<LogicalSlot> logicalSlotsToRelease = new ArrayList<>(allocatedLogicalSlots.values());
    for (LogicalSlot allocatedLogicalSlot : logicalSlotsToRelease) {
        // this will also cause the logical slot to be returned
        allocatedLogicalSlot.releaseSlot(cause);
    }
    allocatedLogicalSlots.clear();
    state = State.RELEASED;
}

Example 33 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class SlotSharingSlotAllocator method tryReserveResources.

@Override
public Optional<ReservedSlots> tryReserveResources(VertexParallelism vertexParallelism) {
    Preconditions.checkArgument(vertexParallelism instanceof VertexParallelismWithSlotSharing, String.format("%s expects %s as argument.", SlotSharingSlotAllocator.class.getSimpleName(), VertexParallelismWithSlotSharing.class.getSimpleName()));
    final VertexParallelismWithSlotSharing vertexParallelismWithSlotSharing = (VertexParallelismWithSlotSharing) vertexParallelism;
    final Collection<AllocationID> expectedSlots = calculateExpectedSlots(vertexParallelismWithSlotSharing.getAssignments());
    if (areAllExpectedSlotsAvailableAndFree(expectedSlots)) {
        final Map<ExecutionVertexID, LogicalSlot> assignedSlots = new HashMap<>();
        for (ExecutionSlotSharingGroupAndSlot executionSlotSharingGroup : vertexParallelismWithSlotSharing.getAssignments()) {
            final SharedSlot sharedSlot = reserveSharedSlot(executionSlotSharingGroup.getSlotInfo());
            for (ExecutionVertexID executionVertexId : executionSlotSharingGroup.getExecutionSlotSharingGroup().getContainedExecutionVertices()) {
                final LogicalSlot logicalSlot = sharedSlot.allocateLogicalSlot();
                assignedSlots.put(executionVertexId, logicalSlot);
            }
        }
        return Optional.of(ReservedSlots.create(assignedSlots));
    } else {
        return Optional.empty();
    }
}

Example 34 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class DefaultExecutionGraphDeploymentWithSmallBlobCacheSizeLimitTest method testDeployMultipleTasksWithSmallBlobCacheSizeLimit.

/**
 * Test the deployment works well even the size limit of {@link BlobCacheSizeTracker} in {@link
 * PermanentBlobCache} is set to the minimum value.
 *
 * <p>In this extreme case, since the size limit is 1, every time a task is deployed, all the
 * existing **tracked** BLOBs on the cache must be untracked and deleted before the new BLOB is
 * stored onto the cache.
 *
 * <p>This extreme case covers the situation of the normal case, where the size limit is much
 * larger than 1 and the deletion won't happen so frequently.
 */
@Test
public void testDeployMultipleTasksWithSmallBlobCacheSizeLimit() throws Exception {
    final int numberOfVertices = 4;
    final int parallelism = 10;
    final ExecutionGraph eg = createAndSetupExecutionGraph(numberOfVertices, parallelism);
    final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    final BlockingQueue<TaskDeploymentDescriptor> tdds = new ArrayBlockingQueue<>(numberOfVertices * parallelism);
    taskManagerGateway.setSubmitConsumer(FunctionUtils.uncheckedConsumer(taskDeploymentDescriptor -> {
        taskDeploymentDescriptor.loadBigData(blobCache);
        tdds.offer(taskDeploymentDescriptor);
    }));
    for (ExecutionJobVertex ejv : eg.getVerticesTopologically()) {
        for (ExecutionVertex ev : ejv.getTaskVertices()) {
            assertEquals(ExecutionState.CREATED, ev.getExecutionState());
            LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway).createTestingLogicalSlot();
            final Execution execution = ev.getCurrentExecutionAttempt();
            execution.transitionState(ExecutionState.SCHEDULED);
            execution.registerProducedPartitions(slot.getTaskManagerLocation(), true).get();
            ev.deployToSlot(slot);
            assertEquals(ExecutionState.DEPLOYING, ev.getExecutionState());
            TaskDeploymentDescriptor tdd = tdds.take();
            assertNotNull(tdd);
            List<InputGateDeploymentDescriptor> igdds = tdd.getInputGates();
            assertEquals(ev.getAllConsumedPartitionGroups().size(), igdds.size());
            if (igdds.size() > 0) {
                checkShuffleDescriptors(igdds.get(0), ev.getConsumedPartitionGroup(0));
            }
        }
    }
}

Example 35 with LogicalSlot

use of org.apache.flink.runtime.jobmaster.LogicalSlot in project flink by apache.

the class ExecutionVertexDeploymentTest method testDeployFailedSynchronous.

@Test
public void testDeployFailedSynchronous() {
    try {
        final ExecutionVertex vertex = getExecutionVertex();
        final LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(new SubmitFailingSimpleAckingTaskManagerGateway()).createTestingLogicalSlot();
        assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
        vertex.getCurrentExecutionAttempt().transitionState(ExecutionState.SCHEDULED);
        vertex.deployToSlot(slot);
        assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
        assertTrue(vertex.getFailureInfo().isPresent());
        assertThat(vertex.getFailureInfo().map(ErrorInfo::getExceptionAsString).get(), containsString(ERROR_MESSAGE));
        assertTrue(vertex.getStateTimestamp(ExecutionState.CREATED) > 0);
        assertTrue(vertex.getStateTimestamp(ExecutionState.DEPLOYING) > 0);
        assertTrue(vertex.getStateTimestamp(ExecutionState.FAILED) > 0);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}