Examples with ArchivedExecutionVertex org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex used on opensource projects

Example 16 with ArchivedExecutionVertex

use of org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex in project flink by apache.

the class SubtaskCurrentAttemptDetailsHandlerTest method testHandleRequest.

@Test
public void testHandleRequest() throws Exception {
    // Prepare the execution graph.
    final JobID jobID = new JobID();
    final JobVertexID jobVertexID = new JobVertexID();
    // The testing subtask.
    final long deployingTs = System.currentTimeMillis() - 1024;
    final long finishedTs = System.currentTimeMillis();
    final long bytesIn = 1L;
    final long bytesOut = 10L;
    final long recordsIn = 20L;
    final long recordsOut = 30L;
    final IOMetrics ioMetrics = new IOMetrics(bytesIn, bytesOut, recordsIn, recordsOut);
    final long[] timestamps = new long[ExecutionState.values().length];
    timestamps[ExecutionState.DEPLOYING.ordinal()] = deployingTs;
    final ExecutionState expectedState = ExecutionState.FINISHED;
    timestamps[expectedState.ordinal()] = finishedTs;
    final LocalTaskManagerLocation assignedResourceLocation = new LocalTaskManagerLocation();
    final AllocationID allocationID = new AllocationID();
    final int subtaskIndex = 1;
    final int attempt = 2;
    final ArchivedExecution execution = new ArchivedExecution(new StringifiedAccumulatorResult[0], ioMetrics, new ExecutionAttemptID(), attempt, expectedState, null, assignedResourceLocation, allocationID, subtaskIndex, timestamps);
    final ArchivedExecutionVertex executionVertex = new ArchivedExecutionVertex(subtaskIndex, "Test archived execution vertex", execution, new EvictingBoundedList<>(0));
    // Instance the handler.
    final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(new Configuration());
    final MetricFetcher metricFetcher = new MetricFetcherImpl<>(() -> null, address -> null, TestingUtils.defaultExecutor(), Time.milliseconds(1000L), MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
    final SubtaskCurrentAttemptDetailsHandler handler = new SubtaskCurrentAttemptDetailsHandler(() -> null, Time.milliseconds(100), Collections.emptyMap(), SubtaskCurrentAttemptDetailsHeaders.getInstance(), new DefaultExecutionGraphCache(restHandlerConfiguration.getTimeout(), Time.milliseconds(restHandlerConfiguration.getRefreshInterval())), TestingUtils.defaultExecutor(), metricFetcher);
    final HashMap<String, String> receivedPathParameters = new HashMap<>(2);
    receivedPathParameters.put(JobIDPathParameter.KEY, jobID.toString());
    receivedPathParameters.put(JobVertexIdPathParameter.KEY, jobVertexID.toString());
    final HandlerRequest<EmptyRequestBody> request = HandlerRequest.resolveParametersAndCreate(EmptyRequestBody.getInstance(), new SubtaskMessageParameters(), receivedPathParameters, Collections.emptyMap(), Collections.emptyList());
    // Handle request.
    final SubtaskExecutionAttemptDetailsInfo detailsInfo = handler.handleRequest(request, executionVertex);
    // Verify
    final IOMetricsInfo ioMetricsInfo = new IOMetricsInfo(bytesIn, true, bytesOut, true, recordsIn, true, recordsOut, true);
    final SubtaskExecutionAttemptDetailsInfo expectedDetailsInfo = new SubtaskExecutionAttemptDetailsInfo(subtaskIndex, expectedState, attempt, assignedResourceLocation.getHostname(), deployingTs, finishedTs, finishedTs - deployingTs, ioMetricsInfo, assignedResourceLocation.getResourceID().getResourceIdString());
    assertEquals(expectedDetailsInfo, detailsInfo);
}

Example 17 with ArchivedExecutionVertex

use of org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex in project flink by apache.

the class SubtaskExecutionAttemptDetailsHandlerTest method testHandleRequest.

@Test
public void testHandleRequest() throws Exception {
    final JobID jobID = new JobID();
    final JobVertexID jobVertexId = new JobVertexID();
    // The testing subtask.
    final int subtaskIndex = 1;
    final ExecutionState expectedState = ExecutionState.FINISHED;
    final int attempt = 0;
    final StringifiedAccumulatorResult[] emptyAccumulators = new StringifiedAccumulatorResult[0];
    final long bytesIn = 1L;
    final long bytesOut = 10L;
    final long recordsIn = 20L;
    final long recordsOut = 30L;
    final IOMetrics ioMetrics = new IOMetrics(bytesIn, bytesOut, recordsIn, recordsOut);
    final ArchivedExecutionJobVertex archivedExecutionJobVertex = new ArchivedExecutionJobVertex(new ArchivedExecutionVertex[] { // the first subtask won't be queried
    null, new ArchivedExecutionVertex(subtaskIndex, "test task", new ArchivedExecution(emptyAccumulators, ioMetrics, new ExecutionAttemptID(), attempt, expectedState, null, null, null, subtaskIndex, new long[ExecutionState.values().length]), new EvictingBoundedList<>(0)) }, jobVertexId, "test", 1, 1, ResourceProfile.UNKNOWN, emptyAccumulators);
    // Change some fields so we can make it different from other sub tasks.
    final MetricFetcher metricFetcher = new MetricFetcherImpl<>(() -> null, address -> null, TestingUtils.defaultExecutor(), Time.milliseconds(1000L), MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
    // Instance the handler.
    final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(new Configuration());
    final SubtaskExecutionAttemptDetailsHandler handler = new SubtaskExecutionAttemptDetailsHandler(() -> null, Time.milliseconds(100L), Collections.emptyMap(), SubtaskExecutionAttemptDetailsHeaders.getInstance(), new DefaultExecutionGraphCache(restHandlerConfiguration.getTimeout(), Time.milliseconds(restHandlerConfiguration.getRefreshInterval())), TestingUtils.defaultExecutor(), metricFetcher);
    final HashMap<String, String> receivedPathParameters = new HashMap<>(4);
    receivedPathParameters.put(JobIDPathParameter.KEY, jobID.toString());
    receivedPathParameters.put(JobVertexIdPathParameter.KEY, jobVertexId.toString());
    receivedPathParameters.put(SubtaskIndexPathParameter.KEY, Integer.toString(subtaskIndex));
    receivedPathParameters.put(SubtaskAttemptPathParameter.KEY, Integer.toString(attempt));
    final HandlerRequest<EmptyRequestBody> request = HandlerRequest.resolveParametersAndCreate(EmptyRequestBody.getInstance(), new SubtaskAttemptMessageParameters(), receivedPathParameters, Collections.emptyMap(), Collections.emptyList());
    // Handle request.
    final SubtaskExecutionAttemptDetailsInfo detailsInfo = handler.handleRequest(request, archivedExecutionJobVertex);
    // Verify
    final IOMetricsInfo ioMetricsInfo = new IOMetricsInfo(bytesIn, true, bytesOut, true, recordsIn, true, recordsOut, true);
    final SubtaskExecutionAttemptDetailsInfo expectedDetailsInfo = new SubtaskExecutionAttemptDetailsInfo(subtaskIndex, expectedState, attempt, "(unassigned)", -1L, 0L, -1L, ioMetricsInfo, "(unassigned)");
    assertEquals(expectedDetailsInfo, detailsInfo);
}

Example 18 with ArchivedExecutionVertex

use of org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex in project flink by apache.

the class DefaultSchedulerTest method failJobIfCannotRestart.

@Test
public void failJobIfCannotRestart() throws Exception {
    final JobGraph jobGraph = singleNonParallelJobVertexJobGraph();
    testRestartBackoffTimeStrategy.setCanRestart(false);
    final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
    final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices());
    final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
    scheduler.updateTaskExecutionState(createFailedTaskExecutionState(attemptId));
    taskRestartExecutor.triggerScheduledTasks();
    waitForTermination(scheduler);
    final JobStatus jobStatus = scheduler.requestJobStatus();
    assertThat(jobStatus, is(equalTo(JobStatus.FAILED)));
}

Example 19 with ArchivedExecutionVertex

use of org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex in project flink by apache.

the class DefaultSchedulerTest method testRestartVerticesOnFailuresInScheduling.

private void testRestartVerticesOnFailuresInScheduling(Consumer<ExecutionVertexID> actionsToTriggerTaskFailure) throws Exception {
    final int parallelism = 2;
    final JobVertex v1 = createVertex("vertex1", parallelism);
    final JobVertex v2 = createVertex("vertex2", parallelism);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(v1, v2);
    testExecutionSlotAllocator.disableAutoCompletePendingRequests();
    final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory();
    final DefaultScheduler scheduler = createScheduler(jobGraph, ComponentMainThreadExecutorServiceAdapter.forMainThread(), schedulingStrategyFactory, new RestartPipelinedRegionFailoverStrategy.Factory());
    final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy();
    scheduler.startScheduling();
    final ExecutionVertexID vid11 = new ExecutionVertexID(v1.getID(), 0);
    final ExecutionVertexID vid12 = new ExecutionVertexID(v1.getID(), 1);
    final ExecutionVertexID vid21 = new ExecutionVertexID(v2.getID(), 0);
    final ExecutionVertexID vid22 = new ExecutionVertexID(v2.getID(), 1);
    schedulingStrategy.schedule(Arrays.asList(vid11, vid12, vid21, vid22));
    assertThat(testExecutionSlotAllocator.getPendingRequests().keySet(), hasSize(4));
    actionsToTriggerTaskFailure.accept(vid11);
    final Iterator<ArchivedExecutionVertex> vertexIterator = scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices().iterator();
    final ArchivedExecutionVertex ev11 = vertexIterator.next();
    final ArchivedExecutionVertex ev12 = vertexIterator.next();
    final ArchivedExecutionVertex ev21 = vertexIterator.next();
    final ArchivedExecutionVertex ev22 = vertexIterator.next();
    // ev11 and ev21 needs to be restarted because it is pipelined region failover and
    // they are in the same region. ev12 and ev22 will not be affected
    assertThat(testExecutionSlotAllocator.getPendingRequests().keySet(), hasSize(2));
    assertThat(ev11.getExecutionState(), is(ExecutionState.FAILED));
    assertThat(ev21.getExecutionState(), is(ExecutionState.CANCELED));
    assertThat(ev12.getExecutionState(), is(ExecutionState.SCHEDULED));
    assertThat(ev22.getExecutionState(), is(ExecutionState.SCHEDULED));
    taskRestartExecutor.triggerScheduledTasks();
    assertThat(schedulingStrategy.getReceivedVerticesToRestart(), containsInAnyOrder(vid11, vid21));
}

Example 20 with ArchivedExecutionVertex

use of org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex in project flink by apache.

the class DefaultSchedulerTest method testLateRegisteredPartitionsWillBeReleased.

@Test
public void testLateRegisteredPartitionsWillBeReleased() {
    shuffleMaster.setAutoCompleteRegistration(false);
    final List<ResultPartitionID> trackedPartitions = new ArrayList<>();
    partitionTracker.setStartTrackingPartitionsConsumer((resourceID, resultPartitionDeploymentDescriptor) -> trackedPartitions.add(resultPartitionDeploymentDescriptor.getShuffleDescriptor().getResultPartitionID()));
    final JobGraph jobGraph = nonParallelSourceSinkJobGraph();
    final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);
    final ArchivedExecutionVertex sourceExecutionVertex = scheduler.requestJob().getArchivedExecutionGraph().getAllExecutionVertices().iterator().next();
    final ExecutionAttemptID attemptId = sourceExecutionVertex.getCurrentExecutionAttempt().getAttemptId();
    scheduler.updateTaskExecutionState(createFailedTaskExecutionState(attemptId));
    // late registered partitions will not be tracked and will be released
    shuffleMaster.completeAllPendingRegistrations();
    assertThat(trackedPartitions, hasSize(0));
    assertThat(shuffleMaster.getExternallyReleasedPartitions(), hasSize(1));
}