Examples with AccumulatorSnapshot org.apache.flink.runtime.accumulators.AccumulatorSnapshot used on opensource projects

Example 1 with AccumulatorSnapshot

use of org.apache.flink.runtime.accumulators.AccumulatorSnapshot in project flink by apache.

the class TaskExecutor method unregisterTaskAndNotifyFinalState.

private void unregisterTaskAndNotifyFinalState(final UUID jobMasterLeaderId, final JobMasterGateway jobMasterGateway, final ExecutionAttemptID executionAttemptID) {
    Task task = taskSlotTable.removeTask(executionAttemptID);
    if (task != null) {
        if (!task.getExecutionState().isTerminal()) {
            try {
                task.failExternally(new IllegalStateException("Task is being remove from TaskManager."));
            } catch (Exception e) {
                log.error("Could not properly fail task.", e);
            }
        }
        log.info("Un-registering task and sending final execution state {} to JobManager for task {} {}.", task.getExecutionState(), task.getTaskInfo().getTaskName(), task.getExecutionId());
        AccumulatorSnapshot accumulatorSnapshot = task.getAccumulatorRegistry().getSnapshot();
        updateTaskExecutionState(jobMasterLeaderId, jobMasterGateway, new TaskExecutionState(task.getJobID(), task.getExecutionId(), task.getExecutionState(), task.getFailureCause(), accumulatorSnapshot, task.getMetricGroup().getIOMetricGroup().createSnapshot()));
    } else {
        log.error("Cannot find task with ID {} to unregister.", executionAttemptID);
    }
}

Example 2 with AccumulatorSnapshot

use of org.apache.flink.runtime.accumulators.AccumulatorSnapshot in project flink by apache.

the class ExecutionGraphDeploymentTest method testAccumulatorsAndMetricsForwarding.

/**
	 * Verifies that {@link ExecutionGraph#updateState(TaskExecutionState)} updates the accumulators and metrics for an
	 * execution that failed or was canceled.
	 */
@Test
public void testAccumulatorsAndMetricsForwarding() throws Exception {
    final JobVertexID jid1 = new JobVertexID();
    final JobVertexID jid2 = new JobVertexID();
    JobVertex v1 = new JobVertex("v1", jid1);
    JobVertex v2 = new JobVertex("v2", jid2);
    Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> graphAndExecutions = setupExecution(v1, 1, v2, 1);
    ExecutionGraph graph = graphAndExecutions.f0;
    // verify behavior for canceled executions
    Execution execution1 = graphAndExecutions.f1.values().iterator().next();
    IOMetrics ioMetrics = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
    Map<String, Accumulator<?, ?>> accumulators = new HashMap<>();
    accumulators.put("acc", new IntCounter(4));
    AccumulatorSnapshot accumulatorSnapshot = new AccumulatorSnapshot(graph.getJobID(), execution1.getAttemptId(), accumulators);
    TaskExecutionState state = new TaskExecutionState(graph.getJobID(), execution1.getAttemptId(), ExecutionState.CANCELED, null, accumulatorSnapshot, ioMetrics);
    graph.updateState(state);
    assertEquals(ioMetrics, execution1.getIOMetrics());
    assertNotNull(execution1.getUserAccumulators());
    assertEquals(4, execution1.getUserAccumulators().get("acc").getLocalValue());
    // verify behavior for failed executions
    Execution execution2 = graphAndExecutions.f1.values().iterator().next();
    IOMetrics ioMetrics2 = new IOMetrics(0, 0, 0, 0, 0, 0.0, 0.0, 0.0, 0.0, 0.0);
    Map<String, Accumulator<?, ?>> accumulators2 = new HashMap<>();
    accumulators2.put("acc", new IntCounter(8));
    AccumulatorSnapshot accumulatorSnapshot2 = new AccumulatorSnapshot(graph.getJobID(), execution2.getAttemptId(), accumulators2);
    TaskExecutionState state2 = new TaskExecutionState(graph.getJobID(), execution2.getAttemptId(), ExecutionState.FAILED, null, accumulatorSnapshot2, ioMetrics2);
    graph.updateState(state2);
    assertEquals(ioMetrics2, execution2.getIOMetrics());
    assertNotNull(execution2.getUserAccumulators());
    assertEquals(8, execution2.getUserAccumulators().get("acc").getLocalValue());
}

Example 3 with AccumulatorSnapshot

use of org.apache.flink.runtime.accumulators.AccumulatorSnapshot in project flink by apache.

the class ExecutionGraph method deserializeAccumulators.

private Map<String, Accumulator<?, ?>> deserializeAccumulators(TaskExecutionState state) {
    AccumulatorSnapshot serializedAccumulators = state.getAccumulators();
    Map<String, Accumulator<?, ?>> accumulators = null;
    if (serializedAccumulators != null) {
        try {
            accumulators = serializedAccumulators.deserializeUserAccumulators(userClassLoader);
        } catch (Exception e) {
            LOG.error("Failed to deserialize final accumulator results.", e);
        }
    }
    return accumulators;
}