use of org.apache.tez.runtime.api.InputInitializer in project tez by apache.
the class TestRootInputInitializerManager method testSuccessBeforeEvent.
// Order event1 success1, success2, event2
// Primarily a failure scenario, when a Task moves back to running from success
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testSuccessBeforeEvent() throws Exception {
InputDescriptor id = mock(InputDescriptor.class);
InputInitializerDescriptor iid = mock(InputInitializerDescriptor.class);
RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>("InputName", id, iid);
InputInitializer initializer = mock(InputInitializer.class);
InputInitializerContext initializerContext = mock(InputInitializerContext.class);
Vertex vertex = mock(Vertex.class);
StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class);
AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
RootInputInitializerManager.InitializerWrapper initializerWrapper = new RootInputInitializerManager.InitializerWrapper(rootInput, initializer, initializerContext, vertex, stateChangeNotifier, appContext);
ApplicationId appId = ApplicationId.newInstance(1000, 1);
TezDAGID dagId = TezDAGID.getInstance(appId, 1);
TezVertexID srcVertexId = TezVertexID.getInstance(dagId, 2);
TezTaskID srcTaskId1 = TezTaskID.getInstance(srcVertexId, 3);
Vertex srcVertex = mock(Vertex.class);
Task srcTask1 = mock(Task.class);
doReturn(TaskState.RUNNING).when(srcTask1).getState();
doReturn(srcTask1).when(srcVertex).getTask(srcTaskId1.getId());
when(appContext.getCurrentDAG().getVertex(any(String.class))).thenReturn(srcVertex);
String srcVertexName = "srcVertexName";
List<TezEvent> eventList = Lists.newLinkedList();
// First Attempt send event
TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1);
EventMetaData sourceInfo11 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId11);
InputInitializerEvent e1 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
TezEvent te1 = new TezEvent(e1, sourceInfo11);
eventList.add(te1);
initializerWrapper.handleInputInitializerEvents(eventList);
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
eventList.clear();
// First attempt, Task success notification
initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId11.getId());
ArgumentCaptor<List> argumentCaptor = ArgumentCaptor.forClass(List.class);
verify(initializer, times(1)).handleInputInitializerEvent(argumentCaptor.capture());
List<InputInitializerEvent> invokedEvents = argumentCaptor.getValue();
assertEquals(1, invokedEvents.size());
reset(initializer);
TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2);
// 2nd attempt succeeded
initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId());
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
// 2nd attempt send event
EventMetaData sourceInfo12 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId12);
InputInitializerEvent e2 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
TezEvent te2 = new TezEvent(e2, sourceInfo12);
eventList.add(te2);
initializerWrapper.handleInputInitializerEvents(eventList);
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
}
use of org.apache.tez.runtime.api.InputInitializer in project tez by apache.
the class TestRootInputInitializerManager method testEventBeforeSuccess.
// Simple testing. No events if task doesn't succeed.
// Also exercises path where two attempts are reported as successful via the stateChangeNotifier.
// Primarily a failure scenario, when a Task moves back to running from success
// Order event1, success1, event2, success2
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEventBeforeSuccess() throws Exception {
InputDescriptor id = mock(InputDescriptor.class);
InputInitializerDescriptor iid = mock(InputInitializerDescriptor.class);
RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>("InputName", id, iid);
InputInitializer initializer = mock(InputInitializer.class);
InputInitializerContext initializerContext = mock(InputInitializerContext.class);
Vertex vertex = mock(Vertex.class);
StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class);
AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
RootInputInitializerManager.InitializerWrapper initializerWrapper = new RootInputInitializerManager.InitializerWrapper(rootInput, initializer, initializerContext, vertex, stateChangeNotifier, appContext);
ApplicationId appId = ApplicationId.newInstance(1000, 1);
TezDAGID dagId = TezDAGID.getInstance(appId, 1);
TezVertexID srcVertexId = TezVertexID.getInstance(dagId, 2);
TezTaskID srcTaskId1 = TezTaskID.getInstance(srcVertexId, 3);
Vertex srcVertex = mock(Vertex.class);
Task srcTask1 = mock(Task.class);
doReturn(TaskState.RUNNING).when(srcTask1).getState();
doReturn(srcTask1).when(srcVertex).getTask(srcTaskId1.getId());
when(appContext.getCurrentDAG().getVertex(any(String.class))).thenReturn(srcVertex);
String srcVertexName = "srcVertexName";
List<TezEvent> eventList = Lists.newLinkedList();
// First Attempt send event
TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1);
EventMetaData sourceInfo11 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId11);
InputInitializerEvent e1 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
TezEvent te1 = new TezEvent(e1, sourceInfo11);
eventList.add(te1);
initializerWrapper.handleInputInitializerEvents(eventList);
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
eventList.clear();
// First attempt, Task success notification
initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId11.getId());
ArgumentCaptor<List> argumentCaptor = ArgumentCaptor.forClass(List.class);
verify(initializer, times(1)).handleInputInitializerEvent(argumentCaptor.capture());
List<InputInitializerEvent> invokedEvents = argumentCaptor.getValue();
assertEquals(1, invokedEvents.size());
reset(initializer);
// 2nd attempt send event
TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2);
EventMetaData sourceInfo12 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId12);
InputInitializerEvent e2 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
TezEvent te2 = new TezEvent(e2, sourceInfo12);
eventList.add(te2);
initializerWrapper.handleInputInitializerEvents(eventList);
verify(initializer, never()).handleInputInitializerEvent(any(List.class));
eventList.clear();
reset(initializer);
// 2nd attempt succeeded
initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId());
verify(initializer, never()).handleInputInitializerEvent(argumentCaptor.capture());
}
use of org.apache.tez.runtime.api.InputInitializer in project tez by apache.
the class RootInputInitializerManager method runInputInitializers.
public void runInputInitializers(List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>> inputs) throws TezException {
for (RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input : inputs) {
InputInitializerContext context = new TezRootInputInitializerContextImpl(input, vertex, appContext, this);
InputInitializer initializer;
try {
TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), vertex.getVertexId());
initializer = createInitializer(input, context);
} finally {
appContext.getHadoopShim().clearHadoopCallerContext();
}
InitializerWrapper initializerWrapper = new InitializerWrapper(input, initializer, context, vertex, entityStateTracker, appContext);
// Register pending vertex update registrations
List<VertexUpdateRegistrationHolder> vertexUpdateRegistrations = pendingVertexRegistrations.removeAll(input.getName());
if (vertexUpdateRegistrations != null) {
for (VertexUpdateRegistrationHolder h : vertexUpdateRegistrations) {
initializerWrapper.registerForVertexStateUpdates(h.vertexName, h.stateSet);
}
}
initializerMap.put(input.getName(), initializerWrapper);
ListenableFuture<List<Event>> future = executor.submit(new InputInitializerCallable(initializerWrapper, dagUgi, appContext));
Futures.addCallback(future, createInputInitializerCallback(initializerWrapper));
}
}
Aggregations