use of org.apache.flink.runtime.rest.messages.job.UserAccumulator in project flink by apache.
the class JobVertexAccumulatorsHandler method handleRequest.
@Override
protected JobVertexAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException {
StringifiedAccumulatorResult[] accs = jobVertex.getAggregatedUserAccumulatorsStringified();
ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
return new JobVertexAccumulatorsInfo(jobVertex.getJobVertexId().toString(), userAccumulatorList);
}
use of org.apache.flink.runtime.rest.messages.job.UserAccumulator in project flink by apache.
the class JobVertexAccumulatorsInfoTest method getTestResponseInstance.
@Override
protected JobVertexAccumulatorsInfo getTestResponseInstance() throws Exception {
List<UserAccumulator> userAccumulatorList = new ArrayList<>(3);
userAccumulatorList.add(new UserAccumulator("test name1", "test type1", "test value1"));
userAccumulatorList.add(new UserAccumulator("test name2", "test type2", "test value2"));
userAccumulatorList.add(new UserAccumulator("test name3", "test type3", "test value3"));
return new JobVertexAccumulatorsInfo("testId", userAccumulatorList);
}
use of org.apache.flink.runtime.rest.messages.job.UserAccumulator in project flink by apache.
the class SubtaskExecutionAttemptAccumulatorsHandlerTest method testHandleRequest.
@Test
public void testHandleRequest() throws Exception {
// Instance the handler.
final RestHandlerConfiguration restHandlerConfiguration = RestHandlerConfiguration.fromConfiguration(new Configuration());
final SubtaskExecutionAttemptAccumulatorsHandler handler = new SubtaskExecutionAttemptAccumulatorsHandler(() -> null, Time.milliseconds(100L), Collections.emptyMap(), SubtaskExecutionAttemptAccumulatorsHeaders.getInstance(), new DefaultExecutionGraphCache(restHandlerConfiguration.getTimeout(), Time.milliseconds(restHandlerConfiguration.getRefreshInterval())), TestingUtils.defaultExecutor());
// Instance a empty request.
final HandlerRequest<EmptyRequestBody> request = HandlerRequest.create(EmptyRequestBody.getInstance(), new SubtaskAttemptMessageParameters());
final Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>(3);
userAccumulators.put("IntCounter", OptionalFailure.of(new IntCounter(10)));
userAccumulators.put("LongCounter", OptionalFailure.of(new LongCounter(100L)));
userAccumulators.put("Failure", OptionalFailure.ofFailure(new FlinkRuntimeException("Test")));
// Instance the expected result.
final StringifiedAccumulatorResult[] accumulatorResults = StringifiedAccumulatorResult.stringifyAccumulatorResults(userAccumulators);
final int attemptNum = 1;
final int subtaskIndex = 2;
// Instance the tested execution.
final ArchivedExecution execution = new ArchivedExecution(accumulatorResults, null, new ExecutionAttemptID(), attemptNum, ExecutionState.FINISHED, null, null, null, subtaskIndex, new long[ExecutionState.values().length]);
// Invoke tested method.
final SubtaskExecutionAttemptAccumulatorsInfo accumulatorsInfo = handler.handleRequest(request, execution);
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(userAccumulators.size());
for (StringifiedAccumulatorResult accumulatorResult : accumulatorResults) {
userAccumulatorList.add(new UserAccumulator(accumulatorResult.getName(), accumulatorResult.getType(), accumulatorResult.getValue()));
}
final SubtaskExecutionAttemptAccumulatorsInfo expected = new SubtaskExecutionAttemptAccumulatorsInfo(subtaskIndex, attemptNum, execution.getAttemptId().toString(), userAccumulatorList);
// Verify.
assertEquals(expected, accumulatorsInfo);
}
use of org.apache.flink.runtime.rest.messages.job.UserAccumulator in project flink by apache.
the class SubtaskExecutionAttemptAccumulatorsHandler method createAccumulatorInfo.
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) {
final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified();
final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
return new SubtaskExecutionAttemptAccumulatorsInfo(execution.getParallelSubtaskIndex(), execution.getAttemptNumber(), execution.getAttemptId().toString(), userAccumulatorList);
}
use of org.apache.flink.runtime.rest.messages.job.UserAccumulator in project flink by apache.
the class SubtasksAllAccumulatorsHandler method handleRequest.
@Override
protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException {
JobVertexID jobVertexId = jobVertex.getJobVertexId();
int parallelism = jobVertex.getParallelism();
final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>();
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {
TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
String locationString = location == null ? "(unassigned)" : location.getHostname();
StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified();
List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length);
for (StringifiedAccumulatorResult acc : accs) {
userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue()));
}
subtaskAccumulatorsInfos.add(new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo(vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(), vertex.getCurrentExecutionAttempt().getAttemptNumber(), locationString, userAccumulators));
}
return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos);
}
Aggregations