Examples with Event org.apache.hadoop.yarn.event.Event used on opensource projects

Example 31 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestTaskAttemptListenerImpl method testGetTask.

@Test(timeout = 5000)
public void testGetTask() throws IOException {
    AppContext appCtx = mock(AppContext.class);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(ea);
    when(appCtx.getEventHandler()).thenReturn(ea);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    MockTaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, hbHandler, policy);
    Configuration conf = new Configuration();
    listener.init(conf);
    listener.start();
    JVMId id = new JVMId("foo", 1, true, 1);
    WrappedJvmID wid = new WrappedJvmID(id.getJobId(), id.isMap, id.getId());
    // Verify ask before registration.
    //The JVM ID has not been registered yet so we should kill it.
    JvmContext context = new JvmContext();
    context.jvmId = id;
    JvmTask result = listener.getTask(context);
    assertNotNull(result);
    assertTrue(result.shouldDie);
    // Verify ask after registration but before launch. 
    // Don't kill, should be null.
    TaskAttemptId attemptID = mock(TaskAttemptId.class);
    Task task = mock(Task.class);
    //Now put a task with the ID
    listener.registerPendingTask(task, wid);
    result = listener.getTask(context);
    assertNull(result);
    // Unregister for more testing.
    listener.unregister(attemptID, wid);
    // Verify ask after registration and launch
    //Now put a task with the ID
    listener.registerPendingTask(task, wid);
    listener.registerLaunchedTask(attemptID, wid);
    verify(hbHandler).register(attemptID);
    result = listener.getTask(context);
    assertNotNull(result);
    assertFalse(result.shouldDie);
    // Don't unregister yet for more testing.
    //Verify that if we call it again a second time we are told to die.
    result = listener.getTask(context);
    assertNotNull(result);
    assertTrue(result.shouldDie);
    listener.unregister(attemptID, wid);
    // Verify after unregistration.
    result = listener.getTask(context);
    assertNotNull(result);
    assertTrue(result.shouldDie);
    listener.stop();
    // test JVMID
    JVMId jvmid = JVMId.forName("jvm_001_002_m_004");
    assertNotNull(jvmid);
    try {
        JVMId.forName("jvm_001_002_m_004_006");
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals(e.getMessage(), "TaskId string : jvm_001_002_m_004_006 is not properly formed");
    }
}

Example 32 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestCheckpointPreemptionPolicy method setup.

@Before
// mocked generics
@SuppressWarnings("rawtypes")
public void setup() {
    ApplicationId appId = ApplicationId.newInstance(200, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    jid = MRBuilderUtils.newJobId(appId, 1);
    mActxt = mock(RunningAppContext.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(mActxt.getEventHandler()).thenReturn(ea);
    for (int i = 0; i < 40; ++i) {
        ContainerId cId = ContainerId.newContainerId(appAttemptId, i);
        if (0 == i % 7) {
            preemptedContainers.add(cId);
        }
        TaskId tId = 0 == i % 2 ? MRBuilderUtils.newTaskId(jid, i / 2, TaskType.MAP) : MRBuilderUtils.newTaskId(jid, i / 2 + 1, TaskType.REDUCE);
        assignedContainers.put(cId, MRBuilderUtils.newTaskAttemptId(tId, 0));
        contToResourceMap.put(cId, Resource.newInstance(2 * minAlloc, 2));
    }
    for (Map.Entry<ContainerId, TaskAttemptId> ent : assignedContainers.entrySet()) {
        System.out.println("cont:" + ent.getKey().getContainerId() + " type:" + ent.getValue().getTaskId().getTaskType() + " res:" + contToResourceMap.get(ent.getKey()).getMemorySize() + "MB");
    }
}

Example 33 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestTaskAttempt method testSingleRackRequest.

@Test
public void testSingleRackRequest() throws Exception {
    TaskAttemptImpl.RequestContainerTransition rct = new TaskAttemptImpl.RequestContainerTransition(false);
    EventHandler eventHandler = mock(EventHandler.class);
    String[] hosts = new String[3];
    hosts[0] = "host1";
    hosts[1] = "host2";
    hosts[2] = "host3";
    TaskSplitMetaInfo splitInfo = new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);
    TaskAttemptImpl mockTaskAttempt = createMapTaskAttemptImplForTest(eventHandler, splitInfo);
    TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);
    rct.transition(mockTaskAttempt, mockTAEvent);
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(eventHandler, times(2)).handle(arg.capture());
    if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
        Assert.fail("Second Event not of type ContainerRequestEvent");
    }
    ContainerRequestEvent cre = (ContainerRequestEvent) arg.getAllValues().get(1);
    String[] requestedRacks = cre.getRacks();
    //Only a single occurrence of /DefaultRack
    assertEquals(1, requestedRacks.length);
}

Example 34 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestCommitterEventHandler method testFailure.

@Test
public void testFailure() throws Exception {
    AppContext mockContext = mock(AppContext.class);
    OutputCommitter mockCommitter = mock(OutputCommitter.class);
    Clock mockClock = mock(Clock.class);
    CommitterEventHandler handler = new CommitterEventHandler(mockContext, mockCommitter, new TestingRMHeartbeatHandler());
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
    JobContext mockJobContext = mock(JobContext.class);
    ApplicationAttemptId attemptid = ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
    JobId jobId = TypeConverter.toYarn(TypeConverter.fromYarn(attemptid.getApplicationId()));
    WaitForItHandler waitForItHandler = new WaitForItHandler();
    when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
    when(mockContext.getApplicationAttemptId()).thenReturn(attemptid);
    when(mockContext.getEventHandler()).thenReturn(waitForItHandler);
    when(mockContext.getClock()).thenReturn(mockClock);
    doThrow(new YarnRuntimeException("Intentional Failure")).when(mockCommitter).commitJob(any(JobContext.class));
    handler.init(conf);
    handler.start();
    try {
        handler.handle(new CommitterJobCommitEvent(jobId, mockJobContext));
        String user = UserGroupInformation.getCurrentUser().getShortUserName();
        Path startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
        Path endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, jobId);
        Path endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, jobId);
        Event e = waitForItHandler.getAndClearEvent();
        assertNotNull(e);
        assertTrue(e instanceof JobCommitFailedEvent);
        FileSystem fs = FileSystem.get(conf);
        assertTrue(fs.exists(startCommitFile));
        assertFalse(fs.exists(endCommitSuccessFile));
        assertTrue(fs.exists(endCommitFailureFile));
        verify(mockCommitter).commitJob(any(JobContext.class));
    } finally {
        handler.stop();
    }
}

Example 35 with Event

use of org.apache.hadoop.yarn.event.Event in project hadoop by apache.

the class TestContainerLauncherImpl method testContainerCleaned.

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test(timeout = 5000)
public void testContainerCleaned() throws Exception {
    LOG.info("STARTING testContainerCleaned");
    CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
    CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);
    AppContext mockContext = mock(AppContext.class);
    EventHandler mockEventHandler = mock(EventHandler.class);
    when(mockContext.getEventHandler()).thenReturn(mockEventHandler);
    ContainerManagementProtocolClient mockCM = new ContainerManagerForTest(startLaunchBarrier, completeLaunchBarrier);
    ContainerLauncherImplUnderTest ut = new ContainerLauncherImplUnderTest(mockContext, mockCM);
    Configuration conf = new Configuration();
    ut.init(conf);
    ut.start();
    try {
        ContainerId contId = makeContainerId(0l, 0, 0, 1);
        TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
        String cmAddress = "127.0.0.1:8000";
        StartContainersResponse startResp = recordFactory.newRecordInstance(StartContainersResponse.class);
        startResp.setAllServicesMetaData(serviceResponse);
        LOG.info("inserting launch event");
        ContainerRemoteLaunchEvent mockLaunchEvent = mock(ContainerRemoteLaunchEvent.class);
        when(mockLaunchEvent.getType()).thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
        when(mockLaunchEvent.getContainerID()).thenReturn(contId);
        when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
        when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
        when(mockLaunchEvent.getContainerToken()).thenReturn(createNewContainerToken(contId, cmAddress));
        ut.handle(mockLaunchEvent);
        startLaunchBarrier.await();
        LOG.info("inserting cleanup event");
        ContainerLauncherEvent mockCleanupEvent = mock(ContainerLauncherEvent.class);
        when(mockCleanupEvent.getType()).thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
        when(mockCleanupEvent.getContainerID()).thenReturn(contId);
        when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
        when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
        ut.handle(mockCleanupEvent);
        completeLaunchBarrier.await();
        ut.waitForPoolToIdle();
        ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
        verify(mockEventHandler, atLeast(2)).handle(arg.capture());
        boolean containerCleaned = false;
        for (int i = 0; i < arg.getAllValues().size(); i++) {
            LOG.info(arg.getAllValues().get(i).toString());
            Event currentEvent = arg.getAllValues().get(i);
            if (currentEvent.getType() == TaskAttemptEventType.TA_CONTAINER_CLEANED) {
                containerCleaned = true;
            }
        }
        assert (containerCleaned);
    } finally {
        ut.stop();
    }
}