Examples with ControlledClock org.apache.hadoop.yarn.util.ControlledClock used on opensource projects

Example 1 with ControlledClock

use of org.apache.hadoop.yarn.util.ControlledClock in project hadoop by apache.

the class TestFairScheduler method testDumpState.

@Test
public void testDumpState() throws IOException {
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"parent\">");
    out.println("  <queue name=\"child1\">");
    out.println("    <weight>1</weight>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    ControlledClock clock = new ControlledClock();
    scheduler.setClock(clock);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    FSLeafQueue child1 = scheduler.getQueueManager().getLeafQueue("parent.child1", false);
    Resource resource = Resource.newInstance(4 * GB, 4);
    child1.setMaxShare(resource);
    FSAppAttempt app = mock(FSAppAttempt.class);
    Mockito.when(app.getDemand()).thenReturn(resource);
    Mockito.when(app.getResourceUsage()).thenReturn(resource);
    child1.addAppSchedulable(app);
    child1.updateDemand();
    String childQueueString = "{Name: root.parent.child1," + " Weight: <memory weight=1.0, cpu weight=1.0>," + " Policy: fair," + " FairShare: <memory:0, vCores:0>," + " SteadyFairShare: <memory:0, vCores:0>," + " MaxShare: <memory:4096, vCores:4>," + " MinShare: <memory:0, vCores:0>," + " ResourceUsage: <memory:4096, vCores:4>," + " Demand: <memory:4096, vCores:4>," + " Runnable: 1," + " NumPendingApps: 0," + " NonRunnable: 0," + " MaxAMShare: 0.5," + " MaxAMResource: <memory:0, vCores:0>," + " AMResourceUsage: <memory:0, vCores:0>," + " LastTimeAtMinShare: " + clock.getTime() + "}";
    assertTrue(child1.dumpState().equals(childQueueString));
    FSParentQueue parent = scheduler.getQueueManager().getParentQueue("parent", false);
    parent.setMaxShare(resource);
    parent.updateDemand();
    String parentQueueString = "{Name: root.parent," + " Weight: <memory weight=1.0, cpu weight=1.0>," + " Policy: fair," + " FairShare: <memory:0, vCores:0>," + " SteadyFairShare: <memory:0, vCores:0>," + " MaxShare: <memory:4096, vCores:4>," + " MinShare: <memory:0, vCores:0>," + " ResourceUsage: <memory:4096, vCores:4>," + " Demand: <memory:4096, vCores:4>," + " MaxAMShare: 0.5," + " Runnable: 0}";
    assertTrue(parent.dumpState().equals(parentQueueString + ", " + childQueueString));
}

Example 2 with ControlledClock

use of org.apache.hadoop.yarn.util.ControlledClock in project hadoop by apache.

the class TestFSAppAttempt method testDelaySchedulingForContinuousScheduling.

@Test
public void testDelaySchedulingForContinuousScheduling() throws InterruptedException {
    FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue("queue", true);
    Priority pri = Mockito.mock(Priority.class);
    SchedulerRequestKey prio = TestUtils.toSchedulerKey(pri);
    Mockito.when(pri.getPriority()).thenReturn(1);
    ControlledClock clock = new ControlledClock();
    scheduler.setClock(clock);
    // 5 seconds
    long nodeLocalityDelayMs = 5 * 1000L;
    // 6 seconds
    long rackLocalityDelayMs = 6 * 1000L;
    RMContext rmContext = resourceManager.getRMContext();
    ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
    FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
    // Default level should be node-local
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // after 4 seconds should remain node local
    clock.tickSec(4);
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // after 6 seconds should switch to rack local
    clock.tickSec(2);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // manually set back to node local
    schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
    schedulerApp.resetSchedulingOpportunities(prio, clock.getTime());
    assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    // Now escalate again to rack-local, then to off-switch
    clock.tickSec(6);
    assertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
    clock.tickSec(7);
    assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
}

Example 3 with ControlledClock

use of org.apache.hadoop.yarn.util.ControlledClock in project hadoop by apache.

the class TestAllocationFileLoaderService method testReload.

@Test(timeout = 10000)
public void testReload() throws Exception {
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("  <queue name=\"queueA\">");
    out.println("    <maxRunningApps>1</maxRunningApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"queueB\" />");
    out.println("  <queuePlacementPolicy>");
    out.println("    <rule name='default' />");
    out.println("  </queuePlacementPolicy>");
    out.println("</allocations>");
    out.close();
    ControlledClock clock = new ControlledClock();
    clock.setTime(0);
    Configuration conf = new Configuration();
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(clock);
    allocLoader.reloadIntervalMs = 5;
    allocLoader.init(conf);
    ReloadListener confHolder = new ReloadListener();
    allocLoader.setReloadListener(confHolder);
    allocLoader.reloadAllocations();
    AllocationConfiguration allocConf = confHolder.allocConf;
    // Verify conf
    QueuePlacementPolicy policy = allocConf.getPlacementPolicy();
    List<QueuePlacementRule> rules = policy.getRules();
    assertEquals(1, rules.size());
    assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass());
    assertEquals(1, allocConf.getQueueMaxApps("root.queueA"));
    assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
    assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF).contains("root.queueA"));
    assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF).contains("root.queueB"));
    confHolder.allocConf = null;
    // Modify file and advance the clock
    out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("  <queue name=\"queueB\">");
    out.println("    <maxRunningApps>3</maxRunningApps>");
    out.println("  </queue>");
    out.println("  <queuePlacementPolicy>");
    out.println("    <rule name='specified' />");
    out.println("    <rule name='nestedUserQueue' >");
    out.println("         <rule name='primaryGroup' />");
    out.println("    </rule>");
    out.println("    <rule name='default' />");
    out.println("  </queuePlacementPolicy>");
    out.println("</allocations>");
    out.close();
    clock.tickMsec(System.currentTimeMillis() + AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000);
    allocLoader.start();
    while (confHolder.allocConf == null) {
        Thread.sleep(20);
    }
    // Verify conf
    allocConf = confHolder.allocConf;
    policy = allocConf.getPlacementPolicy();
    rules = policy.getRules();
    assertEquals(3, rules.size());
    assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass());
    assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1).getClass());
    assertEquals(QueuePlacementRule.PrimaryGroup.class, ((NestedUserQueue) (rules.get(1))).nestedRule.getClass());
    assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass());
    assertEquals(3, allocConf.getQueueMaxApps("root.queueB"));
    assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
    assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF).contains("root.queueB"));
}

Example 4 with ControlledClock

use of org.apache.hadoop.yarn.util.ControlledClock in project hadoop by apache.

the class TestSpeculativeExecutionWithMRApp method testSepculateSuccessfulWithUpdateEvents.

@Test
public void testSepculateSuccessfulWithUpdateEvents() throws Exception {
    Clock actualClock = SystemClock.getInstance();
    final ControlledClock clock = new ControlledClock(actualClock);
    clock.setTime(System.currentTimeMillis());
    MRApp app = new MRApp(NUM_MAPPERS, NUM_REDUCERS, false, "test", true, clock);
    Job job = app.submit(new Configuration(), true, true);
    app.waitForState(job, JobState.RUNNING);
    Map<TaskId, Task> tasks = job.getTasks();
    Assert.assertEquals("Num tasks is not correct", NUM_MAPPERS + NUM_REDUCERS, tasks.size());
    Iterator<Task> taskIter = tasks.values().iterator();
    while (taskIter.hasNext()) {
        app.waitForState(taskIter.next(), TaskState.RUNNING);
    }
    // Process the update events
    clock.setTime(System.currentTimeMillis() + 1000);
    EventHandler appEventHandler = app.getContext().getEventHandler();
    for (Map.Entry<TaskId, Task> mapTask : tasks.entrySet()) {
        for (Map.Entry<TaskAttemptId, TaskAttempt> taskAttempt : mapTask.getValue().getAttempts().entrySet()) {
            TaskAttemptStatus status = createTaskAttemptStatus(taskAttempt.getKey(), (float) 0.5, TaskAttemptState.RUNNING);
            TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status);
            appEventHandler.handle(event);
        }
    }
    Task speculatedTask = null;
    int numTasksToFinish = NUM_MAPPERS + NUM_REDUCERS - 1;
    clock.setTime(System.currentTimeMillis() + 1000);
    for (Map.Entry<TaskId, Task> task : tasks.entrySet()) {
        for (Map.Entry<TaskAttemptId, TaskAttempt> taskAttempt : task.getValue().getAttempts().entrySet()) {
            if (numTasksToFinish > 0) {
                appEventHandler.handle(new TaskAttemptEvent(taskAttempt.getKey(), TaskAttemptEventType.TA_DONE));
                appEventHandler.handle(new TaskAttemptEvent(taskAttempt.getKey(), TaskAttemptEventType.TA_CONTAINER_COMPLETED));
                numTasksToFinish--;
                app.waitForState(taskAttempt.getValue(), TaskAttemptState.SUCCEEDED);
            } else {
                // The last task is chosen for speculation
                TaskAttemptStatus status = createTaskAttemptStatus(taskAttempt.getKey(), (float) 0.75, TaskAttemptState.RUNNING);
                speculatedTask = task.getValue();
                TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status);
                appEventHandler.handle(event);
            }
        }
    }
    clock.setTime(System.currentTimeMillis() + 15000);
    for (Map.Entry<TaskId, Task> task : tasks.entrySet()) {
        for (Map.Entry<TaskAttemptId, TaskAttempt> taskAttempt : task.getValue().getAttempts().entrySet()) {
            if (taskAttempt.getValue().getState() != TaskAttemptState.SUCCEEDED) {
                TaskAttemptStatus status = createTaskAttemptStatus(taskAttempt.getKey(), (float) 0.75, TaskAttemptState.RUNNING);
                TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(taskAttempt.getKey(), status);
                appEventHandler.handle(event);
            }
        }
    }
    final Task speculatedTaskConst = speculatedTask;
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            if (speculatedTaskConst.getAttempts().size() != 2) {
                clock.setTime(System.currentTimeMillis() + 1000);
                return false;
            } else {
                return true;
            }
        }
    }, 1000, 60000);
    TaskAttempt[] ta = makeFirstAttemptWin(appEventHandler, speculatedTask);
    verifySpeculationMessage(app, ta);
    app.waitForState(Service.STATE.STOPPED);
}

Example 5 with ControlledClock

use of org.apache.hadoop.yarn.util.ControlledClock in project hadoop by apache.

the class TestNodesListManager method testCachedResolver.

@Test
public void testCachedResolver() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    ControlledClock clock = new ControlledClock();
    clock.setTime(0);
    final int CACHE_EXPIRY_INTERVAL_SECS = 30;
    NodesListManager.CachedResolver resolver = new NodesListManager.CachedResolver(clock, CACHE_EXPIRY_INTERVAL_SECS);
    resolver.init(new YarnConfiguration());
    resolver.start();
    resolver.addToCache("testCachedResolverHost1", "1.1.1.1");
    Assert.assertEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    resolver.addToCache("testCachedResolverHost2", "1.1.1.2");
    Assert.assertEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    Assert.assertEquals("1.1.1.2", resolver.resolve("testCachedResolverHost2"));
    // test removeFromCache
    resolver.removeFromCache("testCachedResolverHost1");
    Assert.assertNotEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    Assert.assertEquals("1.1.1.2", resolver.resolve("testCachedResolverHost2"));
    // test expiry
    clock.tickMsec(CACHE_EXPIRY_INTERVAL_SECS * 1000 + 1);
    resolver.getExpireChecker().run();
    Assert.assertNotEquals("1.1.1.1", resolver.resolve("testCachedResolverHost1"));
    Assert.assertNotEquals("1.1.1.2", resolver.resolve("testCachedResolverHost2"));
}