Examples with MiniYARNCluster org.apache.hadoop.yarn.server.MiniYARNCluster used on opensource projects

Example 11 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project hadoop by apache.

the class TestHedgingRequestRMFailoverProxyProvider method testHedgingRequestProxyProvider.

@Test
public void testHedgingRequestProxyProvider() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
    conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
    conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
    conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3,rm4,rm5");
    conf.set(YarnConfiguration.CLIENT_FAILOVER_PROXY_PROVIDER, RequestHedgingRMFailoverProxyProvider.class.getName());
    conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 2000);
    try (MiniYARNCluster cluster = new MiniYARNCluster("testHedgingRequestProxyProvider", 5, 0, 1, 1)) {
        HATestUtil.setRpcAddressForRM("rm1", 10000, conf);
        HATestUtil.setRpcAddressForRM("rm2", 20000, conf);
        HATestUtil.setRpcAddressForRM("rm3", 30000, conf);
        HATestUtil.setRpcAddressForRM("rm4", 40000, conf);
        HATestUtil.setRpcAddressForRM("rm5", 50000, conf);
        conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
        cluster.init(conf);
        cluster.start();
        final YarnClient client = YarnClient.createYarnClient();
        client.init(conf);
        client.start();
        // Transition rm5 to active;
        long start = System.currentTimeMillis();
        makeRMActive(cluster, 4);
        validateActiveRM(client);
        long end = System.currentTimeMillis();
        System.out.println("Client call succeeded at " + end);
        // should return the response fast
        Assert.assertTrue(end - start <= 10000);
        // transition rm5 to standby
        cluster.getResourceManager(4).getRMContext().getRMAdminService().transitionToStandby(new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER));
        makeRMActive(cluster, 2);
        validateActiveRM(client);
    }
}

Example 12 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project hadoop by apache.

the class TestYarnCLI method testGetQueueInfoPreemptionDisabled.

@Test
public void testGetQueueInfoPreemptionDisabled() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity." + "ProportionalCapacityPreemptionPolicy");
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    conf.setBoolean(PREFIX + "root.a.a1.disable_preemption", true);
    try (MiniYARNCluster cluster = new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
        YarnClient yarnClient = YarnClient.createYarnClient()) {
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnClient.init(yarnConf);
        yarnClient.start();
        QueueCLI cli = new QueueCLI();
        cli.setClient(yarnClient);
        cli.setSysOutPrintStream(sysOut);
        cli.setSysErrPrintStream(sysErr);
        sysOutStream.reset();
        int result = cli.run(new String[] { "-status", "a1" });
        assertEquals(0, result);
        Assert.assertTrue(sysOutStream.toString().contains("Preemption : disabled"));
    }
}

Example 13 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project hadoop by apache.

the class TestYarnClient method testListReservationsByInvalidTimeInterval.

@Test
public void testListReservationsByInvalidTimeInterval() throws Exception {
    MiniYARNCluster cluster = setupMiniYARNCluster();
    YarnClient client = setupYarnClient(cluster);
    try {
        Clock clock = new UTCClock();
        long arrival = clock.getTime();
        long duration = 60000;
        long deadline = (long) (arrival + 1.05 * duration);
        ReservationSubmissionRequest sRequest = submitReservationTestHelper(client, arrival, deadline, duration);
        // List reservations, search by invalid end time == -1.
        ReservationListRequest request = ReservationListRequest.newInstance(ReservationSystemTestUtil.reservationQ, "", 1, -1, true);
        ReservationListResponse response = client.listReservations(request);
        Assert.assertNotNull(response);
        Assert.assertEquals(1, response.getReservationAllocationState().size());
        Assert.assertEquals(response.getReservationAllocationState().get(0).getReservationId().getId(), sRequest.getReservationId().getId());
        // List reservations, search by invalid end time < -1.
        request = ReservationListRequest.newInstance(ReservationSystemTestUtil.reservationQ, "", 1, -10, true);
        response = client.listReservations(request);
        Assert.assertNotNull(response);
        Assert.assertEquals(1, response.getReservationAllocationState().size());
        Assert.assertEquals(response.getReservationAllocationState().get(0).getReservationId().getId(), sRequest.getReservationId().getId());
    } finally {
        // clean-up
        if (client != null) {
            client.stop();
        }
        cluster.stop();
    }
}

Example 14 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project hadoop by apache.

the class TestYarnCLI method testGetQueueInfoPreemptionEnabled.

@Test
public void testGetQueueInfoPreemptionEnabled() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    ReservationSystemTestUtil.setupQueueConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity." + "ProportionalCapacityPreemptionPolicy");
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
    MiniYARNCluster cluster = new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
    YarnClient yarnClient = null;
    try {
        cluster.init(conf);
        cluster.start();
        final Configuration yarnConf = cluster.getConfig();
        yarnClient = YarnClient.createYarnClient();
        yarnClient.init(yarnConf);
        yarnClient.start();
        QueueCLI cli = new QueueCLI();
        cli.setClient(yarnClient);
        cli.setSysOutPrintStream(sysOut);
        cli.setSysErrPrintStream(sysErr);
        sysOutStream.reset();
        int result = cli.run(new String[] { "-status", "a1" });
        assertEquals(0, result);
        Assert.assertTrue(sysOutStream.toString().contains("Preemption : enabled"));
    } finally {
        // clean-up
        if (yarnClient != null) {
            yarnClient.stop();
        }
        cluster.stop();
        cluster.close();
    }
}

Example 15 with MiniYARNCluster

use of org.apache.hadoop.yarn.server.MiniYARNCluster in project hadoop by apache.

the class TestDistributedShell method setupInternal.

private void setupInternal(int numNodeManager, float timelineVersion) throws Exception {
    LOG.info("Starting up YARN cluster");
    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    // reduce the teardown waiting time
    conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
    conf.set("yarn.log.dir", "target");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    // mark if we need to launch the v1 timeline server
    // disable aux-service based timeline aggregators
    conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
    conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
    conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    conf.set("mapreduce.jobhistory.address", "0.0.0.0:" + ServerSocketUtil.getPort(10021, 10));
    // Enable ContainersMonitorImpl
    conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, LinuxResourceCalculatorPlugin.class.getName());
    conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, ProcfsBasedProcessTree.class.getName());
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
    conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
    // ATS version specific settings
    if (timelineVersion == 1.0f) {
        conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f);
        conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT);
    } else if (timelineVersion == 1.5f) {
        if (hdfsCluster == null) {
            HdfsConfiguration hdfsConfig = new HdfsConfiguration();
            hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig).numDataNodes(1).build();
        }
        fs = hdfsCluster.getFileSystem();
        PluginStoreTestUtils.prepareFileSystemForPluginStore(fs);
        PluginStoreTestUtils.prepareConfiguration(conf, hdfsCluster);
        conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES, DistributedShellTimelinePlugin.class.getName());
    } else if (timelineVersion == 2.0f) {
        // set version to 2
        conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
        // disable v1 timeline server since we no longer have a server here
        // enable aux-service based timeline aggregators
        conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
        conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
        conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter.class);
        timelineV2StorageDir = tmpFolder.newFolder().getAbsolutePath();
        // set the file system timeline writer storage directory
        conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT, timelineV2StorageDir);
    } else {
        Assert.fail("Wrong timeline version number: " + timelineVersion);
    }
    if (yarnCluster == null) {
        yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1, numNodeManager, 1, 1);
        yarnCluster.init(conf);
        yarnCluster.start();
        conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort());
        waitForNMsToRegister();
        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, new File(url.getPath()).getParent());
        //write the document to a buffer (not directly to the file, as that
        //can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        //write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
    }
    FileContext fsContext = FileContext.getLocalFSFileContext();
    fsContext.delete(new Path(conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH)), true);
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
        LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
    }
}