Examples with EventHandler org.apache.hadoop.yarn.event.EventHandler used on opensource projects

Example 46 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestDAGSchedulerNaturalOrderControlled method testMultipleRequestsForSameTask.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testMultipleRequestsForSameTask() {
    EventHandler eventHandler = mock(EventHandler.class);
    DAG dag = createMockDag();
    DAGSchedulerNaturalOrderControlled dagScheduler = new DAGSchedulerNaturalOrderControlled(dag, eventHandler);
    int numVertices = 5;
    Vertex[] vertices = new Vertex[numVertices];
    for (int i = 0; i < numVertices; i++) {
        vertices[i] = dag.getVertex("vertex" + i);
    }
    // Schedule all but 1 task belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 0));
    }
    verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule all tasks belonging to v2
    for (int i = 0; i < vertices[2].getTotalTasks(); i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0));
    }
    // Nothing should be scheduled
    verify(eventHandler, never()).handle(any(Event.class));
    reset(eventHandler);
    // Schedule an extra attempt for all but 1 task belonging to v0
    for (int i = 0; i < vertices[0].getTotalTasks() - 1; i++) {
        dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), i, 1));
    }
    // Only v0 requests should have gone out
    verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any(Event.class));
    reset(eventHandler);
    // Schedule last task of v0, with attempt 1
    dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[0].getVertexId(), vertices[0].getTotalTasks() - 1, 1));
    // One v0 request and all of v2 should have gone out
    verify(eventHandler, times(1 + vertices[2].getTotalTasks())).handle(any(Event.class));
}

Example 47 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestTaskAttempt method testPriority.

@Test(timeout = 5000)
public void testPriority() {
    TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition();
    EventHandler eventHandler = mock(EventHandler.class);
    TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
    TaskAttemptImpl taImplReScheduled = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx, true, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    TaskAttemptEventSchedule sEvent = mock(TaskAttemptEventSchedule.class);
    when(sEvent.getPriorityLowLimit()).thenReturn(3);
    when(sEvent.getPriorityHighLimit()).thenReturn(1);
    sta.transition(taImpl, sEvent);
    verify(eventHandler, times(1)).handle(arg.capture());
    AMSchedulerEventTALaunchRequest launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue();
    Assert.assertEquals(2, launchEvent.getPriority());
    sta.transition(taImplReScheduled, sEvent);
    verify(eventHandler, times(2)).handle(arg.capture());
    launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue();
    Assert.assertEquals(1, launchEvent.getPriority());
    when(sEvent.getPriorityLowLimit()).thenReturn(6);
    when(sEvent.getPriorityHighLimit()).thenReturn(4);
    sta.transition(taImpl, sEvent);
    verify(eventHandler, times(3)).handle(arg.capture());
    launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue();
    Assert.assertEquals(5, launchEvent.getPriority());
    sta.transition(taImplReScheduled, sEvent);
    verify(eventHandler, times(4)).handle(arg.capture());
    launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue();
    Assert.assertEquals(4, launchEvent.getPriority());
    when(sEvent.getPriorityLowLimit()).thenReturn(5);
    when(sEvent.getPriorityHighLimit()).thenReturn(5);
    sta.transition(taImpl, sEvent);
    verify(eventHandler, times(5)).handle(arg.capture());
    launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue();
    Assert.assertEquals(5, launchEvent.getPriority());
    sta.transition(taImplReScheduled, sEvent);
    verify(eventHandler, times(6)).handle(arg.capture());
    launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue();
    Assert.assertEquals(5, launchEvent.getPriority());
}

Example 48 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestTaskAttempt method testHostResolveAttempt.

@Test(timeout = 5000)
public // TODO Move to the client post TEZ-125.
void testHostResolveAttempt() throws Exception {
    TaskAttemptImpl.ScheduleTaskattemptTransition sta = new TaskAttemptImpl.ScheduleTaskattemptTransition();
    EventHandler eventHandler = mock(EventHandler.class);
    String[] hosts = new String[] { "127.0.0.1", "host2", "host3" };
    Set<String> resolved = new TreeSet<String>(Arrays.asList(new String[] { "host1", "host2", "host3" }));
    locationHint = TaskLocationHint.createTaskLocationHint(new TreeSet<String>(Arrays.asList(hosts)), null);
    TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1);
    TaskAttemptImpl taImpl = new MockTaskAttemptImpl(taskID, 1, eventHandler, mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx, false, Resource.newInstance(1024, 1), createFakeContainerContext(), false);
    TaskAttemptImpl spyTa = spy(taImpl);
    when(spyTa.resolveHosts(hosts)).thenReturn(resolved.toArray(new String[3]));
    TaskAttemptEventSchedule mockTAEvent = mock(TaskAttemptEventSchedule.class);
    sta.transition(spyTa, mockTAEvent);
    verify(spyTa).resolveHosts(hosts);
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(eventHandler, times(1)).handle(arg.capture());
    if (!(arg.getAllValues().get(0) instanceof AMSchedulerEventTALaunchRequest)) {
        fail("Second Event not of type ContainerRequestEvent");
    }
    Map<String, Boolean> expected = new HashMap<String, Boolean>();
    expected.put("host1", true);
    expected.put("host2", true);
    expected.put("host3", true);
    Set<String> requestedHosts = spyTa.taskHosts;
    for (String h : requestedHosts) {
        expected.remove(h);
    }
    assertEquals(0, expected.size());
}

Example 49 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestEdge method testInvalidSourceTaskIndex.

@Test(timeout = 5000)
public void testInvalidSourceTaskIndex() throws Exception {
    EventHandler mockEventHandler = mock(EventHandler.class);
    Edge edge = new Edge(EdgeProperty.create(EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName()).setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1, 1, 1, -1).toUserPayload()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(""), InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
    TezVertexID v1Id = createVertexID(1);
    TezVertexID v2Id = createVertexID(2);
    edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
    edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
    edge.initialize();
    try {
        TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1), new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1", TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
        edge.sendTezEventToSourceTasks(ireEvent);
        Assert.fail();
    } catch (AMUserCodeException e) {
        e.printStackTrace();
        assertTrue(e.getCause().getMessage().contains("SourceTaskIndex should not be negative"));
    }
}

Example 50 with EventHandler

use of org.apache.hadoop.yarn.event.EventHandler in project tez by apache.

the class TestEdge method testCompositeEventHandling.

@SuppressWarnings({ "rawtypes" })
@Test(timeout = 5000)
public void testCompositeEventHandling() throws TezException {
    EventHandler eventHandler = mock(EventHandler.class);
    EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class));
    Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration());
    TezVertexID srcVertexID = createVertexID(1);
    TezVertexID destVertexID = createVertexID(2);
    LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
    LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
    TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
    Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
    Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
    edge.setSourceVertex(srcVertex);
    edge.setDestinationVertex(destVertex);
    edge.initialize();
    // Task0, Attempt 0
    TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2);
    EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
    // Verification via a CompositeEvent
    CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes()));
    // AttemptNum
    cdmEvent.setVersion(2);
    TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
    // Event setup to look like it would after the Vertex is done with it.
    edge.sendTezEventToDestinationTasks(tezEvent);
    verifyEvents(srcTAID, destTasks);
    // Same Verification via regular DataMovementEvents
    // Reset the mock
    resetTaskMocks(destTasks.values());
    for (int i = 0; i < destTasks.size(); i++) {
        DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes()));
        dmEvent.setVersion(2);
        tezEvent = new TezEvent(dmEvent, srcMeta);
        edge.sendTezEventToDestinationTasks(tezEvent);
    }
    verifyEvents(srcTAID, destTasks);
}