Examples with TezDAGID org.apache.tez.dag.records.TezDAGID used on opensource projects

Example 31 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestRootInputInitializerManager method testSuccessBeforeEvent.

// Order event1 success1, success2, event2
// Primarily a failure scenario, when a Task moves back to running from success
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testSuccessBeforeEvent() throws Exception {
    InputDescriptor id = mock(InputDescriptor.class);
    InputInitializerDescriptor iid = mock(InputInitializerDescriptor.class);
    RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>("InputName", id, iid);
    InputInitializer initializer = mock(InputInitializer.class);
    InputInitializerContext initializerContext = mock(InputInitializerContext.class);
    Vertex vertex = mock(Vertex.class);
    StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class);
    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    RootInputInitializerManager.InitializerWrapper initializerWrapper = new RootInputInitializerManager.InitializerWrapper(rootInput, initializer, initializerContext, vertex, stateChangeNotifier, appContext);
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID srcVertexId = TezVertexID.getInstance(dagId, 2);
    TezTaskID srcTaskId1 = TezTaskID.getInstance(srcVertexId, 3);
    Vertex srcVertex = mock(Vertex.class);
    Task srcTask1 = mock(Task.class);
    doReturn(TaskState.RUNNING).when(srcTask1).getState();
    doReturn(srcTask1).when(srcVertex).getTask(srcTaskId1.getId());
    when(appContext.getCurrentDAG().getVertex(any(String.class))).thenReturn(srcVertex);
    String srcVertexName = "srcVertexName";
    List<TezEvent> eventList = Lists.newLinkedList();
    // First Attempt send event
    TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1);
    EventMetaData sourceInfo11 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId11);
    InputInitializerEvent e1 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
    TezEvent te1 = new TezEvent(e1, sourceInfo11);
    eventList.add(te1);
    initializerWrapper.handleInputInitializerEvents(eventList);
    verify(initializer, never()).handleInputInitializerEvent(any(List.class));
    eventList.clear();
    // First attempt, Task success notification
    initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId11.getId());
    ArgumentCaptor<List> argumentCaptor = ArgumentCaptor.forClass(List.class);
    verify(initializer, times(1)).handleInputInitializerEvent(argumentCaptor.capture());
    List<InputInitializerEvent> invokedEvents = argumentCaptor.getValue();
    assertEquals(1, invokedEvents.size());
    reset(initializer);
    TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2);
    // 2nd attempt succeeded
    initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId());
    verify(initializer, never()).handleInputInitializerEvent(any(List.class));
    // 2nd attempt send event
    EventMetaData sourceInfo12 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId12);
    InputInitializerEvent e2 = InputInitializerEvent.create("fakeVertex", "fakeInput", null);
    TezEvent te2 = new TezEvent(e2, sourceInfo12);
    eventList.add(te2);
    initializerWrapper.handleInputInitializerEvents(eventList);
    verify(initializer, never()).handleInputInitializerEvent(any(List.class));
}

Example 32 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestTaskSchedulerManager method testTaskSchedulerRouting.

@Test(timeout = 5000)
public void testTaskSchedulerRouting() throws Exception {
    Configuration conf = new Configuration(false);
    UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf);
    String customSchedulerName = "fakeScheduler";
    List<NamedEntityDescriptor> taskSchedulers = new LinkedList<>();
    ByteBuffer bb = ByteBuffer.allocate(4);
    bb.putInt(0, 3);
    UserPayload userPayload = UserPayload.create(bb);
    taskSchedulers.add(new NamedEntityDescriptor(customSchedulerName, FakeTaskScheduler.class.getName()).setUserPayload(userPayload));
    taskSchedulers.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null).setUserPayload(defaultPayload));
    TSEHForMultipleSchedulersTest tseh = new TSEHForMultipleSchedulersTest(mockAppContext, mockClientService, mockEventHandler, mockSigMatcher, mockWebUIService, taskSchedulers, false);
    tseh.init(conf);
    tseh.start();
    // Verify that the YARN task scheduler is installed by default
    assertTrue(tseh.getYarnSchedulerCreated());
    assertFalse(tseh.getUberSchedulerCreated());
    assertEquals(2, tseh.getNumCreateInvocations());
    // Verify the order of the schedulers
    assertEquals(customSchedulerName, tseh.getTaskSchedulerName(0));
    assertEquals(TezConstants.getTezYarnServicePluginName(), tseh.getTaskSchedulerName(1));
    verify(tseh.getTestTaskScheduler(0)).initialize();
    verify(tseh.getTestTaskScheduler(0)).start();
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId1 = TezTaskID.getInstance(vertexID, 1);
    TezTaskAttemptID attemptId11 = TezTaskAttemptID.getInstance(taskId1, 1);
    TezTaskID taskId2 = TezTaskID.getInstance(vertexID, 2);
    TezTaskAttemptID attemptId21 = TezTaskAttemptID.getInstance(taskId2, 1);
    Resource resource = Resource.newInstance(1024, 1);
    TaskAttempt mockTaskAttempt1 = mock(TaskAttempt.class);
    TaskAttempt mockTaskAttempt2 = mock(TaskAttempt.class);
    AMSchedulerEventTALaunchRequest launchRequest1 = new AMSchedulerEventTALaunchRequest(attemptId11, resource, mock(TaskSpec.class), mockTaskAttempt1, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 0, 0, 0);
    tseh.handle(launchRequest1);
    verify(tseh.getTestTaskScheduler(0)).allocateTask(eq(mockTaskAttempt1), eq(resource), any(String[].class), any(String[].class), any(Priority.class), any(Object.class), eq(launchRequest1));
    AMSchedulerEventTALaunchRequest launchRequest2 = new AMSchedulerEventTALaunchRequest(attemptId21, resource, mock(TaskSpec.class), mockTaskAttempt2, mock(TaskLocationHint.class), 1, mock(ContainerContext.class), 1, 0, 0);
    tseh.handle(launchRequest2);
    verify(tseh.getTestTaskScheduler(1)).allocateTask(eq(mockTaskAttempt2), eq(resource), any(String[].class), any(String[].class), any(Priority.class), any(Object.class), eq(launchRequest2));
}

Example 33 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestStateChangeNotifier method testEventsOnRegistration.

@Test(timeout = 5000)
public void testEventsOnRegistration() {
    TezDAGID dagId = TezDAGID.getInstance("1", 1, 1);
    Vertex v1 = createMockVertex(dagId, 1);
    Vertex v2 = createMockVertex(dagId, 2);
    Vertex v3 = createMockVertex(dagId, 3);
    DAG dag = createMockDag(dagId, v1, v2, v3);
    StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
    // Vertex has sent one event
    notifyTracker(tracker, v1, VertexState.RUNNING);
    VertexStateUpdateListener mockListener11 = mock(VertexStateUpdateListener.class);
    VertexStateUpdateListener mockListener12 = mock(VertexStateUpdateListener.class);
    VertexStateUpdateListener mockListener13 = mock(VertexStateUpdateListener.class);
    VertexStateUpdateListener mockListener14 = mock(VertexStateUpdateListener.class);
    // Register for all states
    tracker.registerForVertexUpdates(v1.getName(), null, mockListener11);
    // Register for all states
    tracker.registerForVertexUpdates(v1.getName(), EnumSet.allOf(VertexState.class), mockListener12);
    // Register for specific state, event generated
    tracker.registerForVertexUpdates(v1.getName(), EnumSet.of(VertexState.RUNNING), mockListener13);
    // Register for specific state, event not generated
    tracker.registerForVertexUpdates(v1.getName(), EnumSet.of(VertexState.SUCCEEDED), mockListener14);
    ArgumentCaptor<VertexStateUpdate> argumentCaptor = ArgumentCaptor.forClass(VertexStateUpdate.class);
    verify(mockListener11, times(1)).onStateUpdated(argumentCaptor.capture());
    assertEquals(VertexState.RUNNING, argumentCaptor.getValue().getVertexState());
    verify(mockListener12, times(1)).onStateUpdated(argumentCaptor.capture());
    assertEquals(VertexState.RUNNING, argumentCaptor.getValue().getVertexState());
    verify(mockListener13, times(1)).onStateUpdated(argumentCaptor.capture());
    assertEquals(VertexState.RUNNING, argumentCaptor.getValue().getVertexState());
    verify(mockListener14, never()).onStateUpdated(any(VertexStateUpdate.class));
    // Vertex has not notified of state
    tracker.reset();
    VertexStateUpdateListener mockListener2 = mock(VertexStateUpdateListener.class);
    tracker.registerForVertexUpdates(v2.getName(), null, mockListener2);
    // there should no be any event sent out
    Assert.assertEquals(0, tracker.totalCount.get());
    verify(mockListener2, never()).onStateUpdated(any(VertexStateUpdate.class));
    // Vertex has notified about parallelism update only
    tracker.stateChanged(v3.getVertexId(), new VertexStateUpdateParallelismUpdated(v3.getName(), 23, -1));
    VertexStateUpdateListener mockListener3 = mock(VertexStateUpdateListener.class);
    tracker.registerForVertexUpdates(v3.getName(), null, mockListener3);
    verify(mockListener3, times(1)).onStateUpdated(argumentCaptor.capture());
    assertEquals(VertexState.PARALLELISM_UPDATED, argumentCaptor.getValue().getVertexState());
}

Example 34 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestStateChangeNotifier method testSimpleStateUpdates.

@Test(timeout = 5000)
public void testSimpleStateUpdates() {
    TezDAGID dagId = TezDAGID.getInstance("1", 1, 1);
    Vertex v1 = createMockVertex(dagId, 1);
    DAG dag = createMockDag(dagId, v1);
    StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
    VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
    tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
    List<VertexState> expectedStates = Lists.newArrayList(VertexState.RUNNING, VertexState.SUCCEEDED, VertexState.FAILED, VertexState.KILLED, VertexState.RUNNING, VertexState.SUCCEEDED);
    for (VertexState state : expectedStates) {
        notifyTracker(tracker, v1, state);
    }
    ArgumentCaptor<VertexStateUpdate> argumentCaptor = ArgumentCaptor.forClass(VertexStateUpdate.class);
    verify(mockListener, times(expectedStates.size())).onStateUpdated(argumentCaptor.capture());
    List<VertexStateUpdate> stateUpdatesSent = argumentCaptor.getAllValues();
    Iterator<VertexState> expectedStateIter = expectedStates.iterator();
    for (int i = 0; i < expectedStates.size(); i++) {
        assertEquals(expectedStateIter.next(), stateUpdatesSent.get(i).getVertexState());
    }
}

Example 35 with TezDAGID

use of org.apache.tez.dag.records.TezDAGID in project tez by apache.

the class TestStateChangeNotifier method testDuplicateRegistration.

@Test(timeout = 5000)
public void testDuplicateRegistration() {
    TezDAGID dagId = TezDAGID.getInstance("1", 1, 1);
    Vertex v1 = createMockVertex(dagId, 1);
    DAG dag = createMockDag(dagId, v1);
    StateChangeNotifierForTest tracker = new StateChangeNotifierForTest(dag);
    VertexStateUpdateListener mockListener = mock(VertexStateUpdateListener.class);
    tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
    try {
        tracker.registerForVertexUpdates(v1.getName(), null, mockListener);
        fail("Expecting an error from duplicate registrations of the same listener");
    } catch (TezUncheckedException e) {
    // Expected, ignore
    }
}