Examples with LocalTaskManagerLocation org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation used on opensource projects

Example 11 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class SlotPoolUtils method offerSlots.

private static ResourceID offerSlots(SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor, List<ResourceProfile> resourceProfiles, TaskManagerGateway taskManagerGateway, boolean assertAllSlotsAreAccepted) {
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    CompletableFuture.runAsync(() -> {
        slotPool.registerTaskManager(taskManagerLocation.getResourceID());
        final Collection<SlotOffer> slotOffers = IntStream.range(0, resourceProfiles.size()).mapToObj(i -> new SlotOffer(new AllocationID(), i, resourceProfiles.get(i))).collect(Collectors.toList());
        final Collection<SlotOffer> acceptedOffers = slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);
        if (assertAllSlotsAreAccepted) {
            assertThat(acceptedOffers, is(slotOffers));
        }
    }, mainThreadExecutor).join();
    return taskManagerLocation.getResourceID();
}

Example 12 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultSchedulerTest method testExceptionHistoryConcurrentRestart.

@Test
public void testExceptionHistoryConcurrentRestart() throws Exception {
    final JobGraph jobGraph = singleJobVertexJobGraph(2);
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingLogicalSlotBuilder logicalSlotBuilder = new TestingLogicalSlotBuilder();
    logicalSlotBuilder.setTaskManagerLocation(taskManagerLocation);
    executionSlotAllocatorFactory = new TestExecutionSlotAllocatorFactory(logicalSlotBuilder);
    final ReorganizableManuallyTriggeredScheduledExecutor delayExecutor = new ReorganizableManuallyTriggeredScheduledExecutor();
    final TestFailoverStrategyFactory failoverStrategyFactory = new TestFailoverStrategyFactory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), new PipelinedRegionSchedulingStrategy.Factory(), failoverStrategyFactory, delayExecutor);
    scheduler.startScheduling();
    final ExecutionVertex executionVertex0 = Iterables.get(scheduler.getExecutionGraph().getAllExecutionVertices(), 0);
    final ExecutionVertex executionVertex1 = Iterables.get(scheduler.getExecutionGraph().getAllExecutionVertices(), 1);
    // single-ExecutionVertex failure
    final RuntimeException exception0 = new RuntimeException("failure #0");
    failoverStrategyFactory.setTasksToRestart(executionVertex0.getID());
    final long updateStateTriggeringRestartTimestamp0 = initiateFailure(scheduler, executionVertex0.getCurrentExecutionAttempt().getAttemptId(), exception0);
    // multi-ExecutionVertex failure
    final RuntimeException exception1 = new RuntimeException("failure #1");
    failoverStrategyFactory.setTasksToRestart(executionVertex1.getID(), executionVertex0.getID());
    final long updateStateTriggeringRestartTimestamp1 = initiateFailure(scheduler, executionVertex1.getCurrentExecutionAttempt().getAttemptId(), exception1);
    // there might be a race condition with the delayExecutor if the tasks are scheduled quite
    // close to each other which we want to simulate here
    Collections.reverse(delayExecutor.getCollectedScheduledTasks());
    delayExecutor.triggerNonPeriodicScheduledTasks();
    assertThat(scheduler.getExceptionHistory(), IsIterableWithSize.iterableWithSize(2));
    final Iterator<RootExceptionHistoryEntry> actualExceptionHistory = scheduler.getExceptionHistory().iterator();
    final RootExceptionHistoryEntry entry0 = actualExceptionHistory.next();
    assertThat(entry0, is(ExceptionHistoryEntryMatcher.matchesFailure(exception0, updateStateTriggeringRestartTimestamp0, executionVertex0.getTaskNameWithSubtaskIndex(), executionVertex0.getCurrentAssignedResourceLocation())));
    assertThat(entry0.getConcurrentExceptions(), IsIterableContainingInOrder.contains(ExceptionHistoryEntryMatcher.matchesFailure(exception1, updateStateTriggeringRestartTimestamp1, executionVertex1.getTaskNameWithSubtaskIndex(), executionVertex1.getCurrentAssignedResourceLocation())));
    final RootExceptionHistoryEntry entry1 = actualExceptionHistory.next();
    assertThat(entry1, is(ExceptionHistoryEntryMatcher.matchesFailure(exception1, updateStateTriggeringRestartTimestamp1, executionVertex1.getTaskNameWithSubtaskIndex(), executionVertex1.getCurrentAssignedResourceLocation())));
    assertThat(entry1.getConcurrentExceptions(), IsEmptyIterable.emptyIterable());
}

Example 13 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultPreferredLocationsRetrieverTest method testStateLocationsWillBeReturnedIfExist.

@Test
public void testStateLocationsWillBeReturnedIfExist() {
    final TaskManagerLocation stateLocation = new LocalTaskManagerLocation();
    final TestingInputsLocationsRetriever.Builder locationRetrieverBuilder = new TestingInputsLocationsRetriever.Builder();
    final ExecutionVertexID consumerId = new ExecutionVertexID(new JobVertexID(), 0);
    final ExecutionVertexID producerId = new ExecutionVertexID(new JobVertexID(), 0);
    locationRetrieverBuilder.connectConsumerToProducer(consumerId, producerId);
    final TestingInputsLocationsRetriever inputsLocationsRetriever = locationRetrieverBuilder.build();
    inputsLocationsRetriever.markScheduled(producerId);
    final PreferredLocationsRetriever locationsRetriever = new DefaultPreferredLocationsRetriever(id -> Optional.of(stateLocation), inputsLocationsRetriever);
    final CompletableFuture<Collection<TaskManagerLocation>> preferredLocations = locationsRetriever.getPreferredLocations(consumerId, Collections.emptySet());
    assertThat(preferredLocations.getNow(null), contains(stateLocation));
}

Example 14 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class JobExceptionsHandlerTest method testTaskManagerLocationHandling.

@Test
public void testTaskManagerLocationHandling() {
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    assertThat(JobExceptionsHandler.toString(taskManagerLocation), is(String.format("%s:%s", taskManagerLocation.getFQDNHostname(), taskManagerLocation.dataPort())));
}

Example 15 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class SharedSlotTest method testLogicalSlotAllocation.

@Test
public void testLogicalSlotAllocation() {
    CompletableFuture<PhysicalSlot> slotContextFuture = new CompletableFuture<>();
    CompletableFuture<ExecutionSlotSharingGroup> released = new CompletableFuture<>();
    SharedSlot sharedSlot = SharedSlotBuilder.newBuilder().withSlotContextFuture(slotContextFuture).slotWillBeOccupiedIndefinitely().withExternalReleaseCallback(released::complete).build();
    CompletableFuture<LogicalSlot> logicalSlotFuture = sharedSlot.allocateLogicalSlot(EV1);
    assertThat(logicalSlotFuture.isDone(), is(false));
    AllocationID allocationId = new AllocationID();
    LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
    slotContextFuture.complete(new TestingPhysicalSlot(allocationId, taskManagerLocation, 3, taskManagerGateway, RP));
    assertThat(sharedSlot.isEmpty(), is(false));
    assertThat(released.isDone(), is(false));
    assertThat(logicalSlotFuture.isDone(), is(true));
    LogicalSlot logicalSlot = logicalSlotFuture.join();
    assertThat(logicalSlot.getAllocationId(), is(allocationId));
    assertThat(logicalSlot.getTaskManagerLocation(), is(taskManagerLocation));
    assertThat(logicalSlot.getTaskManagerGateway(), is(taskManagerGateway));
    assertThat(logicalSlot.getLocality(), is(Locality.UNKNOWN));
}