Examples with Dispatcher org.apache.hadoop.yarn.event.Dispatcher used on opensource projects

Example 31 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestProportionalCapacityPreemptionPolicy method setup.

@Before
@SuppressWarnings("unchecked")
public void setup() {
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, 1.0f);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, 1.0f);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    // FairScheduler doesn't support this test,
    // Set CapacityScheduler as the scheduler for this test.
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    mClock = mock(Clock.class);
    mCS = mock(CapacityScheduler.class);
    when(mCS.getResourceCalculator()).thenReturn(rc);
    lm = mock(RMNodeLabelsManager.class);
    try {
        when(lm.isExclusiveNodeLabel(anyString())).thenReturn(true);
    } catch (IOException e) {
    // do nothing
    }
    when(mCS.getConfiguration()).thenReturn(conf);
    rmContext = mock(RMContext.class);
    when(mCS.getRMContext()).thenReturn(rmContext);
    when(mCS.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(rmContext.getNodeLabelManager()).thenReturn(lm);
    mDisp = mock(EventHandler.class);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    rand = new Random();
    long seed = rand.nextLong();
    System.out.println(name.getMethodName() + " SEED: " + seed);
    rand.setSeed(seed);
    appAlloc = 0;
}

Example 32 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class ProportionalCapacityPreemptionPolicyMockFramework method setup.

@SuppressWarnings("unchecked")
@Before
public void setup() {
    org.apache.log4j.Logger.getRootLogger().setLevel(org.apache.log4j.Level.DEBUG);
    conf = new CapacitySchedulerConfiguration(new Configuration(false));
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, 10000);
    conf.setLong(CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, 3000);
    // report "ideal" preempt
    conf.setFloat(CapacitySchedulerConfiguration.TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
    conf.setFloat(CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR, (float) 1.0);
    mClock = mock(Clock.class);
    cs = mock(CapacityScheduler.class);
    when(cs.getResourceCalculator()).thenReturn(rc);
    when(cs.getPreemptionManager()).thenReturn(new PreemptionManager());
    when(cs.getConfiguration()).thenReturn(conf);
    nlm = mock(RMNodeLabelsManager.class);
    mDisp = mock(EventHandler.class);
    rmContext = mock(RMContext.class);
    when(rmContext.getNodeLabelManager()).thenReturn(nlm);
    Dispatcher disp = mock(Dispatcher.class);
    when(rmContext.getDispatcher()).thenReturn(disp);
    when(disp.getEventHandler()).thenReturn(mDisp);
    when(cs.getRMContext()).thenReturn(rmContext);
    partitionToResource = new HashMap<>();
    nodeIdToSchedulerNodes = new HashMap<>();
    nameToCSQueues = new HashMap<>();
}

Example 33 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestRMApplicationHistoryWriter method setup.

@Before
public void setup() {
    store = new MemoryApplicationHistoryStore();
    Configuration conf = new Configuration();
    conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
    conf.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
    writer = new RMApplicationHistoryWriter() {

        @Override
        protected ApplicationHistoryStore createApplicationHistoryStore(Configuration conf) {
            return store;
        }

        @Override
        protected Dispatcher createDispatcher(Configuration conf) {
            MultiThreadedDispatcher dispatcher = new MultiThreadedDispatcher(conf.getInt(YarnConfiguration.RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE, YarnConfiguration.DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE));
            dispatcher.setDrainEventsOnStop();
            return dispatcher;
        }

        class MultiThreadedDispatcher extends RMApplicationHistoryWriter.MultiThreadedDispatcher {

            public MultiThreadedDispatcher(int num) {
                super(num);
            }

            @Override
            protected AsyncDispatcher createDispatcher() {
                CounterDispatcher dispatcher = new CounterDispatcher();
                dispatchers.add(dispatcher);
                return dispatcher;
            }
        }
    };
    writer.init(conf);
    writer.start();
}

Example 34 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestAMRMRPCNodeUpdates method setUp.

@Before
public void setUp() {
    dispatcher = new DrainDispatcher();
    this.rm = new MockRM() {

        @Override
        public void init(Configuration conf) {
            conf.set(CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, "1.0");
            super.init(conf);
        }

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm.start();
    amService = rm.getApplicationMasterService();
}

Example 35 with Dispatcher

use of org.apache.hadoop.yarn.event.Dispatcher in project hadoop by apache.

the class TestNodesListManager method testNodeUsableEvent.

@Test(timeout = 300000)
public void testNodeUsableEvent() throws Exception {
    Logger rootLogger = LogManager.getRootLogger();
    rootLogger.setLevel(Level.DEBUG);
    final Dispatcher dispatcher = getDispatcher();
    YarnConfiguration conf = new YarnConfiguration();
    MockRM rm = new MockRM(conf) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 28000);
    NodesListManager nodesListManager = rm.getNodesListManager();
    Resource clusterResource = Resource.newInstance(28000, 8);
    RMNode rmnode = MockNodes.newNodeInfo(1, clusterResource);
    // Create killing APP
    RMApp killrmApp = rm.submitApp(200);
    rm.killApp(killrmApp.getApplicationId());
    rm.waitForState(killrmApp.getApplicationId(), RMAppState.KILLED);
    // Create finish APP
    RMApp finshrmApp = rm.submitApp(2000);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = finshrmApp.getCurrentAppAttempt();
    MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
    am.registerAppAttempt();
    am.unregisterAppAttempt();
    nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    // Create submitted App
    RMApp subrmApp = rm.submitApp(200);
    // Fire Event for NODE_USABLE
    nodesListManager.handle(new NodesListManagerEvent(NodesListManagerEventType.NODE_USABLE, rmnode));
    if (applist.size() > 0) {
        Assert.assertTrue("Event based on running app expected " + subrmApp.getApplicationId(), applist.contains(subrmApp.getApplicationId()));
        Assert.assertFalse("Event based on finish app not expected " + finshrmApp.getApplicationId(), applist.contains(finshrmApp.getApplicationId()));
        Assert.assertFalse("Event based on killed app not expected " + killrmApp.getApplicationId(), applist.contains(killrmApp.getApplicationId()));
    } else {
        Assert.fail("Events received should have beeen more than 1");
    }
    applist.clear();
    // Fire Event for NODE_UNUSABLE
    nodesListManager.handle(new NodesListManagerEvent(NodesListManagerEventType.NODE_UNUSABLE, rmnode));
    if (applist.size() > 0) {
        Assert.assertTrue("Event based on running app expected " + subrmApp.getApplicationId(), applist.contains(subrmApp.getApplicationId()));
        Assert.assertFalse("Event based on finish app not expected " + finshrmApp.getApplicationId(), applist.contains(finshrmApp.getApplicationId()));
        Assert.assertFalse("Event based on killed app not expected " + killrmApp.getApplicationId(), applist.contains(killrmApp.getApplicationId()));
    } else {
        Assert.fail("Events received should have beeen more than 1");
    }
}