Examples with MiniYARNCluster org.apache.hadoop.yarn.server.MiniYARNCluster used on opensource projects

Example 41 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project incubator-gobblin by apache.

the class YarnServiceTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);
    this.clusterConf = new YarnConfiguration();
    this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100");
    this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000");
    this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000");
    this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1));
    this.yarnCluster.init(this.clusterConf);
    this.yarnCluster.start();
    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");
    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(this.clusterConf);
    this.yarnClient.start();
    URL url = YarnServiceTest.class.getClassLoader().getResource(YarnServiceTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);
    this.config = ConfigFactory.parseURL(url).resolve();
    // Start a dummy application manager so that the YarnService can use the AM-RM token.
    startApp();
    // create and start the test yarn service
    this.yarnService = new TestYarnService(this.config, "testApp", "appId", this.clusterConf, FileSystem.getLocal(new Configuration()), this.eventBus);
    this.yarnService.startUp();
}

Example 42 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project incubator-gobblin by apache.

the class YarnServiceTestWithExpiration method setUp.

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);
    this.clusterConf = new YarnConfiguration();
    this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100");
    this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000");
    this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000");
    this.clusterConf.set(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, "1000");
    this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1));
    this.yarnCluster.init(this.clusterConf);
    this.yarnCluster.start();
    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");
    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(this.clusterConf);
    this.yarnClient.start();
    URL url = YarnServiceTest.class.getClassLoader().getResource(YarnServiceTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);
    this.config = ConfigFactory.parseURL(url).resolve();
    // Start a dummy application manager so that the YarnService can use the AM-RM token.
    startApp();
    // create and start the test yarn service
    this.expiredYarnService = new TestExpiredYarnService(this.config, "testApp", "appId", this.clusterConf, FileSystem.getLocal(new Configuration()), this.eventBus);
    this.expiredYarnService.startUp();
}