Examples with YarnClient org.apache.hadoop.yarn.client.api.YarnClient used on opensource projects

Example 11 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestHedgingRequestRMFailoverProxyProvider method testHedgingRequestProxyProvider.

@Test
public void testHedgingRequestProxyProvider() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
    conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
    conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3,rm4,rm5");
    conf.set(YarnConfiguration.CLIENT_FAILOVER_PROXY_PROVIDER, RequestHedgingRMFailoverProxyProvider.class.getName());
    conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 2000);
    try (MiniYARNCluster cluster = new MiniYARNCluster("testHedgingRequestProxyProvider", 5, 0, 1, 1)) {
        HATestUtil.setRpcAddressForRM("rm1", 10000, conf);
        HATestUtil.setRpcAddressForRM("rm2", 20000, conf);
        HATestUtil.setRpcAddressForRM("rm3", 30000, conf);
        HATestUtil.setRpcAddressForRM("rm4", 40000, conf);
        HATestUtil.setRpcAddressForRM("rm5", 50000, conf);
        conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
        cluster.init(conf);
        cluster.start();
        final YarnClient client = YarnClient.createYarnClient();
        client.init(conf);
        client.start();
        // Transition rm5 to active;
        long start = System.currentTimeMillis();
        makeRMActive(cluster, 4);
        validateActiveRM(client);
        long end = System.currentTimeMillis();
        System.out.println("Client call succeeded at " + end);
        // should return the response fast
        Assert.assertTrue(end - start <= 10000);
        // transition rm5 to standby
        cluster.getResourceManager(4).getRMContext().getRMAdminService().transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER));
        makeRMActive(cluster, 2);
        validateActiveRM(client);
    }
}

Example 12 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class LogsCLI method createYarnClient.

@VisibleForTesting
protected YarnClient createYarnClient() {
    YarnClient client = YarnClient.createYarnClient();
    client.init(getConf());
    client.start();
    return client;
}

Example 13 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestMRTimelineEventHandling method testMRNewTimelineServiceEventHandling.

@SuppressWarnings("deprecation")
@Test
public void testMRNewTimelineServiceEventHandling() throws Exception {
    LOG.info("testMRNewTimelineServiceEventHandling start.");
    String testDir = new File("target", getClass().getSimpleName() + "-test_dir").getAbsolutePath();
    String storageDir = testDir + File.separator + "timeline_service_data";
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    // enable new timeline service
    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
    conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class);
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
    // set the file system root directory
    conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, storageDir);
    // enable aux-service based timeline collectors
    conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
    conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
    conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
    MiniMRYarnCluster cluster = null;
    try {
        cluster = new MiniMRYarnCluster(TestMRTimelineEventHandling.class.getSimpleName(), 1, true);
        cluster.init(conf);
        cluster.start();
        LOG.info("A MiniMRYarnCluster get start.");
        Path inDir = new Path(testDir, "input");
        Path outDir = new Path(testDir, "output");
        LOG.info("Run 1st job which should be successful.");
        JobConf successConf = new JobConf(conf);
        successConf.set("dummy_conf1", UtilsForTests.createConfigValue(51 * 1024));
        successConf.set("dummy_conf2", UtilsForTests.createConfigValue(51 * 1024));
        successConf.set("huge_dummy_conf1", UtilsForTests.createConfigValue(101 * 1024));
        successConf.set("huge_dummy_conf2", UtilsForTests.createConfigValue(101 * 1024));
        RunningJob job = UtilsForTests.runJobSucceed(successConf, inDir, outDir);
        Assert.assertEquals(JobStatus.SUCCEEDED, job.getJobStatus().getState().getValue());
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(new Configuration(cluster.getConfig()));
        yarnClient.start();
        EnumSet<YarnApplicationState> appStates = EnumSet.allOf(YarnApplicationState.class);
        ApplicationId firstAppId = null;
        List<ApplicationReport> apps = yarnClient.getApplications(appStates);
        Assert.assertEquals(apps.size(), 1);
        ApplicationReport appReport = apps.get(0);
        firstAppId = appReport.getApplicationId();
        UtilsForTests.waitForAppFinished(job, cluster);
        checkNewTimelineEvent(firstAppId, appReport, storageDir);
        LOG.info("Run 2nd job which should be failed.");
        job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir);
        Assert.assertEquals(JobStatus.FAILED, job.getJobStatus().getState().getValue());
        apps = yarnClient.getApplications(appStates);
        Assert.assertEquals(apps.size(), 2);
        appReport = apps.get(0).getApplicationId().equals(firstAppId) ? apps.get(0) : apps.get(1);
        checkNewTimelineEvent(firstAppId, appReport, storageDir);
    } finally {
        if (cluster != null) {
            cluster.stop();
        }
        // Cleanup test file
        File testDirFolder = new File(testDir);
        if (testDirFolder.isDirectory()) {
            FileUtils.deleteDirectory(testDirFolder);
        }
    }
}

Example 14 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestYarnCLI method testGetQueueInfoPreemptionDisabled.

@Test
public void testGetQueueInfoPreemptionDisabled() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity." + "ProportionalCapacityPreemptionPolicy");
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    conf.setBoolean(PREFIX + "root.a.a1.disable_preemption", true);
    try (MiniYARNCluster cluster = new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
        YarnClient yarnClient = YarnClient.createYarnClient()) {
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnClient.init(yarnConf);
        yarnClient.start();
        QueueCLI cli = new QueueCLI();
        cli.setClient(yarnClient);
        cli.setSysOutPrintStream(sysOut);
        cli.setSysErrPrintStream(sysErr);
        sysOutStream.reset();
        int result = cli.run(new String[] { "-status", "a1" });
        assertEquals(0, result);
        Assert.assertTrue(sysOutStream.toString().contains("Preemption : disabled"));
    }
}

Example 15 with YarnClient

use of org.apache.hadoop.yarn.client.api.YarnClient in project hadoop by apache.

the class TestLogsCLI method testUnknownApplicationId.

@Test(timeout = 5000l)
public void testUnknownApplicationId() throws Exception {
    Configuration conf = new YarnConfiguration();
    YarnClient mockYarnClient = createMockYarnClientUnknownApp();
    LogsCLI cli = new LogsCLIForTest(mockYarnClient);
    cli.setConf(conf);
    int exitCode = cli.run(new String[] { "-applicationId", ApplicationId.newInstance(1, 1).toString() });
    // Error since no logs present for the app.
    assertTrue(exitCode != 0);
    assertTrue(sysErrStream.toString().startsWith("Unable to get ApplicationState"));
}