Examples with LocalTaskManagerLocation org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation used on opensource projects

Example 16 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultExecutionGraphDeploymentTest method testExecutionGraphIsDeployedInTopologicalOrder.

/**
 * Tests that the {@link ExecutionGraph} is deployed in topological order.
 */
@Test
public void testExecutionGraphIsDeployedInTopologicalOrder() throws Exception {
    final int sourceParallelism = 2;
    final int sinkParallelism = 1;
    final JobVertex sourceVertex = new JobVertex("source");
    sourceVertex.setInvokableClass(NoOpInvokable.class);
    sourceVertex.setParallelism(sourceParallelism);
    final JobVertex sinkVertex = new JobVertex("sink");
    sinkVertex.setInvokableClass(NoOpInvokable.class);
    sinkVertex.setParallelism(sinkParallelism);
    sinkVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    final int numberTasks = sourceParallelism + sinkParallelism;
    final ArrayBlockingQueue<ExecutionAttemptID> submittedTasksQueue = new ArrayBlockingQueue<>(numberTasks);
    TestingTaskExecutorGatewayBuilder testingTaskExecutorGatewayBuilder = new TestingTaskExecutorGatewayBuilder();
    testingTaskExecutorGatewayBuilder.setSubmitTaskConsumer((taskDeploymentDescriptor, jobMasterId) -> {
        submittedTasksQueue.offer(taskDeploymentDescriptor.getExecutionAttemptId());
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingTaskExecutorGateway taskExecutorGateway = testingTaskExecutorGatewayBuilder.createTestingTaskExecutorGateway();
    final RpcTaskManagerGateway taskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, JobMasterId.generate());
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(sourceVertex, sinkVertex);
    final TestingPhysicalSlotProvider physicalSlotProvider = TestingPhysicalSlotProvider.createWithoutImmediatePhysicalSlotCreation();
    final SchedulerBase scheduler = SchedulerTestingUtils.newSchedulerBuilder(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread()).setExecutionSlotAllocatorFactory(SchedulerTestingUtils.newSlotSharingExecutionSlotAllocatorFactory(physicalSlotProvider)).setFutureExecutor(new DirectScheduledExecutorService()).build();
    final ExecutionGraph executionGraph = scheduler.getExecutionGraph();
    scheduler.startScheduling();
    // change the order in which the futures are completed
    final List<CompletableFuture<TestingPhysicalSlot>> shuffledFutures = new ArrayList<>(physicalSlotProvider.getResponses().values());
    Collections.shuffle(shuffledFutures);
    for (CompletableFuture<TestingPhysicalSlot> slotFuture : shuffledFutures) {
        slotFuture.complete(TestingPhysicalSlot.builder().withTaskManagerLocation(taskManagerLocation).withTaskManagerGateway(taskManagerGateway).build());
    }
    final List<ExecutionAttemptID> submittedTasks = new ArrayList<>(numberTasks);
    for (int i = 0; i < numberTasks; i++) {
        submittedTasks.add(submittedTasksQueue.take());
    }
    final Collection<ExecutionAttemptID> firstStage = new ArrayList<>(sourceParallelism);
    for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sourceVertex.getID()).getTaskVertices()) {
        firstStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
    }
    final Collection<ExecutionAttemptID> secondStage = new ArrayList<>(sinkParallelism);
    for (ExecutionVertex taskVertex : executionGraph.getJobVertex(sinkVertex.getID()).getTaskVertices()) {
        secondStage.add(taskVertex.getCurrentExecutionAttempt().getAttemptId());
    }
    assertThat(submittedTasks, new ExecutionStageMatcher(Arrays.asList(firstStage, secondStage)));
}

Example 17 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class DefaultSchedulerTest method testExceptionHistoryWithRestartableFailure.

@Test
public void testExceptionHistoryWithRestartableFailure() {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
    final TestingLogicalSlotBuilder logicalSlotBuilder = new TestingLogicalSlotBuilder();
    logicalSlotBuilder.setTaskManagerLocation(taskManagerLocation);
    executionSlotAllocatorFactory = new TestExecutionSlotAllocatorFactory(logicalSlotBuilder);
    final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
    // initiate restartable failure
    final ArchivedExecutionVertex taskFailureExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices());
    final RuntimeException restartableException = new RuntimeException("restartable exception");
    final long updateStateTriggeringRestartTimestamp = initiateFailure(scheduler, taskFailureExecutionVertex.getCurrentExecutionAttempt().getAttemptId(), restartableException);
    taskRestartExecutor.triggerNonPeriodicScheduledTask();
    // initiate job failure
    testRestartBackoffTimeStrategy.setCanRestart(false);
    final ExecutionAttemptID failingAttemptId = Iterables.getOnlyElement(scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices()).getCurrentExecutionAttempt().getAttemptId();
    final RuntimeException failingException = new RuntimeException("failing exception");
    final long updateStateTriggeringJobFailureTimestamp = initiateFailure(scheduler, failingAttemptId, failingException);
    final Iterable<RootExceptionHistoryEntry> actualExceptionHistory = scheduler.getExceptionHistory();
    // assert restarted attempt
    assertThat(actualExceptionHistory, IsIterableContainingInOrder.contains(ExceptionHistoryEntryMatcher.matchesFailure(restartableException, updateStateTriggeringRestartTimestamp, taskFailureExecutionVertex.getTaskNameWithSubtaskIndex(), taskFailureExecutionVertex.getCurrentAssignedResourceLocation()), ExceptionHistoryEntryMatcher.matchesGlobalFailure(failingException, updateStateTriggeringJobFailureTimestamp)));
}

Example 18 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class JobExceptionsHandlerTest method createArchivedExecutionJobVertex.

private static ArchivedExecutionJobVertex createArchivedExecutionJobVertex(JobVertexID jobVertexID) {
    final StringifiedAccumulatorResult[] emptyAccumulators = new StringifiedAccumulatorResult[0];
    final long[] timestamps = new long[ExecutionState.values().length];
    final ExecutionState expectedState = ExecutionState.RUNNING;
    final LocalTaskManagerLocation assignedResourceLocation = new LocalTaskManagerLocation();
    final AllocationID allocationID = new AllocationID();
    final int subtaskIndex = 1;
    final int attempt = 2;
    return new ArchivedExecutionJobVertex(new ArchivedExecutionVertex[] { new ArchivedExecutionVertex(subtaskIndex, "test task", new ArchivedExecution(new StringifiedAccumulatorResult[0], null, new ExecutionAttemptID(), attempt, expectedState, new ErrorInfo(new RuntimeException("error"), System.currentTimeMillis()), assignedResourceLocation, allocationID, subtaskIndex, timestamps), new EvictingBoundedList<>(0)) }, jobVertexID, jobVertexID.toString(), 1, 1, ResourceProfile.UNKNOWN, emptyAccumulators);
}

Example 19 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class JobExceptionsHandlerTest method testWithExceptionHistory.

@Test
public void testWithExceptionHistory() throws HandlerRequestException {
    final RootExceptionHistoryEntry rootCause = fromGlobalFailure(new RuntimeException("exception #0"), System.currentTimeMillis());
    final RootExceptionHistoryEntry otherFailure = new RootExceptionHistoryEntry(new RuntimeException("exception #1"), System.currentTimeMillis(), "task name", new LocalTaskManagerLocation(), Collections.emptySet());
    final ExecutionGraphInfo executionGraphInfo = createExecutionGraphInfo(rootCause, otherFailure);
    final HandlerRequest<EmptyRequestBody> request = createRequest(executionGraphInfo.getJobId(), 10);
    final JobExceptionsInfoWithHistory response = testInstance.handleRequest(request, executionGraphInfo);
    assertThat(response.getExceptionHistory().getEntries(), contains(historyContainsGlobalFailure(rootCause.getException(), rootCause.getTimestamp()), historyContainsJobExceptionInfo(otherFailure.getException(), otherFailure.getTimestamp(), otherFailure.getFailingTaskName(), JobExceptionsHandler.toString(otherFailure.getTaskManagerLocation()))));
    assertFalse(response.getExceptionHistory().isTruncated());
}

Example 20 with LocalTaskManagerLocation

use of org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation in project flink by apache.

the class JobExceptionsHandlerTest method createAccessExecutionGraph.

private static ExecutionGraphInfo createAccessExecutionGraph(int numTasks) {
    Map<JobVertexID, ArchivedExecutionJobVertex> tasks = new HashMap<>();
    for (int i = 0; i < numTasks; i++) {
        final JobVertexID jobVertexId = new JobVertexID();
        tasks.put(jobVertexId, createArchivedExecutionJobVertex(jobVertexId));
    }
    final Throwable failureCause = new RuntimeException("root cause");
    final long failureTimestamp = System.currentTimeMillis();
    final List<RootExceptionHistoryEntry> exceptionHistory = Collections.singletonList(new RootExceptionHistoryEntry(failureCause, failureTimestamp, "test task #1", new LocalTaskManagerLocation(), Collections.emptySet()));
    return new ExecutionGraphInfo(new ArchivedExecutionGraphBuilder().setFailureCause(new ErrorInfo(failureCause, failureTimestamp)).setTasks(tasks).build(), exceptionHistory);
}