Examples with RecordFactory org.apache.hadoop.yarn.factories.RecordFactory used on opensource projects

Example 16 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project tez by apache.

the class TestTezLocalCacheManager method createFile.

// create a temporary file with the given content and return a LocalResource
private static LocalResource createFile(String content) throws IOException {
    FileContext fs = FileContext.getLocalFSFileContext();
    java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
    File temp = tempFile.toFile();
    temp.deleteOnExit();
    Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());
    Files.write(tempFile, content.getBytes());
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
    URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
    ret.setResource(yarnUrlFromPath);
    ret.setSize(content.getBytes().length);
    ret.setType(LocalResourceType.FILE);
    ret.setVisibility(LocalResourceVisibility.PRIVATE);
    ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
    return ret;
}

Example 17 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestCapacityScheduler method testCSReservationWithRootUnblocked.

@Test
public void testCSReservationWithRootUnblocked() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    conf.setResourceComparator(DominantResourceCalculator.class);
    setupOtherBlockedQueueConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    ParentQueue q = (ParentQueue) cs.getQueue("p1");
    Assert.assertNotNull(q);
    String host = "127.0.0.1";
    String host1 = "test";
    RMNode node = MockNodes.newNodeInfo(0, Resource.newInstance(8 * GB, 8), 1, host);
    RMNode node1 = MockNodes.newNodeInfo(0, Resource.newInstance(8 * GB, 8), 2, host1);
    cs.handle(new NodeAddedSchedulerEvent(node));
    cs.handle(new NodeAddedSchedulerEvent(node1));
    ApplicationAttemptId appAttemptId1 = appHelper(rm, cs, 100, 1, "x1", "userX1");
    ApplicationAttemptId appAttemptId2 = appHelper(rm, cs, 100, 2, "x2", "userX2");
    ApplicationAttemptId appAttemptId3 = appHelper(rm, cs, 100, 3, "y1", "userY1");
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    Priority priority = TestUtils.createMockPriority(1);
    ResourceRequest y1Req = null;
    ResourceRequest x1Req = null;
    ResourceRequest x2Req = null;
    for (int i = 0; i < 4; i++) {
        y1Req = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
        cs.allocate(appAttemptId3, Collections.<ResourceRequest>singletonList(y1Req), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
        CapacityScheduler.schedule(cs);
    }
    assertEquals("Y1 Used Resource should be 4 GB", 4 * GB, cs.getQueue("y1").getUsedResources().getMemorySize());
    assertEquals("P2 Used Resource should be 4 GB", 4 * GB, cs.getQueue("p2").getUsedResources().getMemorySize());
    for (int i = 0; i < 7; i++) {
        x1Req = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
        cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(x1Req), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
        CapacityScheduler.schedule(cs);
    }
    assertEquals("X1 Used Resource should be 7 GB", 7 * GB, cs.getQueue("x1").getUsedResources().getMemorySize());
    assertEquals("P1 Used Resource should be 7 GB", 7 * GB, cs.getQueue("p1").getUsedResources().getMemorySize());
    x2Req = TestUtils.createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, priority, recordFactory);
    cs.allocate(appAttemptId2, Collections.<ResourceRequest>singletonList(x2Req), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    CapacityScheduler.schedule(cs);
    assertEquals("X2 Used Resource should be 0", 0, cs.getQueue("x2").getUsedResources().getMemorySize());
    assertEquals("P1 Used Resource should be 7 GB", 7 * GB, cs.getQueue("p1").getUsedResources().getMemorySize());
    //this assign should fail
    x1Req = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
    cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(x1Req), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    CapacityScheduler.schedule(cs);
    assertEquals("X1 Used Resource should be 7 GB", 7 * GB, cs.getQueue("x1").getUsedResources().getMemorySize());
    assertEquals("P1 Used Resource should be 7 GB", 7 * GB, cs.getQueue("p1").getUsedResources().getMemorySize());
    //this should get thru
    for (int i = 0; i < 4; i++) {
        y1Req = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
        cs.allocate(appAttemptId3, Collections.<ResourceRequest>singletonList(y1Req), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
        CapacityScheduler.schedule(cs);
    }
    assertEquals("P2 Used Resource should be 8 GB", 8 * GB, cs.getQueue("p2").getUsedResources().getMemorySize());
    //Free a container from X1
    ContainerId containerId = ContainerId.newContainerId(appAttemptId1, 2);
    cs.handle(new ContainerExpiredSchedulerEvent(containerId));
    //Schedule pending request
    CapacityScheduler.schedule(cs);
    assertEquals("X2 Used Resource should be 2 GB", 2 * GB, cs.getQueue("x2").getUsedResources().getMemorySize());
    assertEquals("P1 Used Resource should be 8 GB", 8 * GB, cs.getQueue("p1").getUsedResources().getMemorySize());
    assertEquals("P2 Used Resource should be 8 GB", 8 * GB, cs.getQueue("p2").getUsedResources().getMemorySize());
    assertEquals("Root Used Resource should be 16 GB", 16 * GB, cs.getRootQueue().getUsedResources().getMemorySize());
    rm.stop();
}

Example 18 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestCapacityScheduler method testCSQueueBlocked.

@Test
public void testCSQueueBlocked() throws Exception {
    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
    setupBlockedQueueConfiguration(conf);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    LeafQueue q = (LeafQueue) cs.getQueue("a");
    Assert.assertNotNull(q);
    String host = "127.0.0.1";
    String host1 = "test";
    RMNode node = MockNodes.newNodeInfo(0, Resource.newInstance(8 * GB, 8), 1, host);
    RMNode node1 = MockNodes.newNodeInfo(0, Resource.newInstance(8 * GB, 8), 2, host1);
    cs.handle(new NodeAddedSchedulerEvent(node));
    cs.handle(new NodeAddedSchedulerEvent(node1));
    //add app begin
    ApplicationAttemptId appAttemptId1 = appHelper(rm, cs, 100, 1, "a", "user1");
    ApplicationAttemptId appAttemptId2 = appHelper(rm, cs, 100, 2, "b", "user2");
    //add app end
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    Priority priority = TestUtils.createMockPriority(1);
    ResourceRequest r1 = TestUtils.createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, priority, recordFactory);
    //This will allocate for app1
    cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(r1), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS).getContainers().size();
    CapacityScheduler.schedule(cs);
    ResourceRequest r2 = null;
    for (int i = 0; i < 13; i++) {
        r2 = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
        cs.allocate(appAttemptId2, Collections.<ResourceRequest>singletonList(r2), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
        CapacityScheduler.schedule(cs);
    }
    assertEquals("A Used Resource should be 2 GB", 2 * GB, cs.getQueue("a").getUsedResources().getMemorySize());
    assertEquals("B Used Resource should be 13 GB", 13 * GB, cs.getQueue("b").getUsedResources().getMemorySize());
    r1 = TestUtils.createResourceRequest(ResourceRequest.ANY, 2 * GB, 1, true, priority, recordFactory);
    r2 = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
    cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(r1), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS).getContainers().size();
    CapacityScheduler.schedule(cs);
    cs.allocate(appAttemptId2, Collections.<ResourceRequest>singletonList(r2), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    CapacityScheduler.schedule(cs);
    //Check blocked Resource
    assertEquals("A Used Resource should be 2 GB", 2 * GB, cs.getQueue("a").getUsedResources().getMemorySize());
    assertEquals("B Used Resource should be 13 GB", 13 * GB, cs.getQueue("b").getUsedResources().getMemorySize());
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId2, 10);
    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId2, 11);
    cs.handle(new ContainerExpiredSchedulerEvent(containerId1));
    rm.drainEvents();
    CapacityScheduler.schedule(cs);
    cs.handle(new ContainerExpiredSchedulerEvent(containerId2));
    CapacityScheduler.schedule(cs);
    rm.drainEvents();
    assertEquals("A Used Resource should be 4 GB", 4 * GB, cs.getQueue("a").getUsedResources().getMemorySize());
    assertEquals("B Used Resource should be 12 GB", 12 * GB, cs.getQueue("b").getUsedResources().getMemorySize());
    assertEquals("Used Resource on Root should be 16 GB", 16 * GB, cs.getRootQueue().getUsedResources().getMemorySize());
    rm.stop();
}

Example 19 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestCapacityScheduler method testAllocateReorder.

@Test
public void testAllocateReorder() throws Exception {
    //Confirm that allocation (resource request) alone will trigger a change in
    //application ordering where appropriate
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    LeafQueue q = (LeafQueue) cs.getQueue("default");
    Assert.assertNotNull(q);
    FairOrderingPolicy fop = new FairOrderingPolicy();
    fop.setSizeBasedWeight(true);
    q.setOrderingPolicy(fop);
    String host = "127.0.0.1";
    RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
    //add app begin
    ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId1, 1);
    RMAppAttemptMetrics attemptMetric1 = new RMAppAttemptMetrics(appAttemptId1, rm.getRMContext());
    RMAppImpl app1 = mock(RMAppImpl.class);
    when(app1.getApplicationId()).thenReturn(appId1);
    RMAppAttemptImpl attempt1 = mock(RMAppAttemptImpl.class);
    Container container = mock(Container.class);
    when(attempt1.getMasterContainer()).thenReturn(container);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(attempt1.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt1.getAppAttemptId()).thenReturn(appAttemptId1);
    when(attempt1.getRMAppAttemptMetrics()).thenReturn(attemptMetric1);
    when(app1.getCurrentAppAttempt()).thenReturn(attempt1);
    rm.getRMContext().getRMApps().put(appId1, app1);
    SchedulerEvent addAppEvent1 = new AppAddedSchedulerEvent(appId1, "default", "user");
    cs.handle(addAppEvent1);
    SchedulerEvent addAttemptEvent1 = new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
    cs.handle(addAttemptEvent1);
    //add app end
    //add app begin
    ApplicationId appId2 = BuilderUtils.newApplicationId(100, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(appId2, 1);
    RMAppAttemptMetrics attemptMetric2 = new RMAppAttemptMetrics(appAttemptId2, rm.getRMContext());
    RMAppImpl app2 = mock(RMAppImpl.class);
    when(app2.getApplicationId()).thenReturn(appId2);
    RMAppAttemptImpl attempt2 = mock(RMAppAttemptImpl.class);
    when(attempt2.getMasterContainer()).thenReturn(container);
    when(attempt2.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt2.getAppAttemptId()).thenReturn(appAttemptId2);
    when(attempt2.getRMAppAttemptMetrics()).thenReturn(attemptMetric2);
    when(app2.getCurrentAppAttempt()).thenReturn(attempt2);
    rm.getRMContext().getRMApps().put(appId2, app2);
    SchedulerEvent addAppEvent2 = new AppAddedSchedulerEvent(appId2, "default", "user");
    cs.handle(addAppEvent2);
    SchedulerEvent addAttemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
    cs.handle(addAttemptEvent2);
    //add app end
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    Priority priority = TestUtils.createMockPriority(1);
    ResourceRequest r1 = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
    //This will allocate for app1
    cs.allocate(appAttemptId1, Collections.<ResourceRequest>singletonList(r1), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    //And this will result in container assignment for app1
    CapacityScheduler.schedule(cs);
    //Verify that app1 is still first in assignment order
    //This happens because app2 has no demand/a magnitude of NaN, which
    //results in app1 and app2 being equal in the fairness comparison and
    //failling back to fifo (start) ordering
    assertEquals(q.getOrderingPolicy().getAssignmentIterator().next().getId(), appId1.toString());
    //Now, allocate for app2 (this would be the first/AM allocation)
    ResourceRequest r2 = TestUtils.createResourceRequest(ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory);
    cs.allocate(appAttemptId2, Collections.<ResourceRequest>singletonList(r2), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    //In this case we do not perform container assignment because we want to
    //verify re-ordering based on the allocation alone
    //Now, the first app for assignment is app2
    assertEquals(q.getOrderingPolicy().getAssignmentIterator().next().getId(), appId2.toString());
    rm.stop();
}

Example 20 with RecordFactory

use of org.apache.hadoop.yarn.factories.RecordFactory in project hadoop by apache.

the class TestOpportunisticContainerAllocatorAMService method testRPCWrapping.

// Test if the OpportunisticContainerAllocatorAMService can handle both
// DSProtocol as well as AMProtocol clients
@Test
public void testRPCWrapping() throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, addr);
    final RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
    final RMContext rmContext = new RMContextImpl() {

        @Override
        public AMLivelinessMonitor getAMLivelinessMonitor() {
            return null;
        }

        @Override
        public Configuration getYarnConfiguration() {
            return new YarnConfiguration();
        }

        @Override
        public RMContainerTokenSecretManager getContainerTokenSecretManager() {
            return new RMContainerTokenSecretManager(conf);
        }
    };
    Container c = factory.newRecordInstance(Container.class);
    c.setExecutionType(ExecutionType.OPPORTUNISTIC);
    c.setId(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(12345, 1), 2), 3));
    AllocateRequest allReq = (AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class);
    allReq.setAskList(Arrays.asList(ResourceRequest.newInstance(Priority.UNDEFINED, "a", Resource.newInstance(1, 2), 1, true, "exp", ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true))));
    OpportunisticContainerAllocatorAMService service = createService(factory, rmContext, c);
    conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
    Server server = service.getServer(rpc, conf, addr, null);
    server.start();
    // Verify that the OpportunisticContainerAllocatorAMSercvice can handle
    // vanilla ApplicationMasterProtocol clients
    RPC.setProtocolEngine(conf, ApplicationMasterProtocolPB.class, ProtobufRpcEngine.class);
    ApplicationMasterProtocolPB ampProxy = RPC.getProxy(ApplicationMasterProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
    RegisterApplicationMasterResponse regResp = new RegisterApplicationMasterResponsePBImpl(ampProxy.registerApplicationMaster(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals("dummyQueue", regResp.getQueue());
    FinishApplicationMasterResponse finishResp = new FinishApplicationMasterResponsePBImpl(ampProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(false, finishResp.getIsUnregistered());
    AllocateResponse allocResp = new AllocateResponsePBImpl(ampProxy.allocate(null, ((AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class)).getProto()));
    List<Container> allocatedContainers = allocResp.getAllocatedContainers();
    Assert.assertEquals(1, allocatedContainers.size());
    Assert.assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainers.get(0).getExecutionType());
    Assert.assertEquals(12345, allocResp.getNumClusterNodes());
    // Verify that the DistrubutedSchedulingService can handle the
    // DistributedSchedulingAMProtocol clients as well
    RPC.setProtocolEngine(conf, DistributedSchedulingAMProtocolPB.class, ProtobufRpcEngine.class);
    DistributedSchedulingAMProtocolPB dsProxy = RPC.getProxy(DistributedSchedulingAMProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
    RegisterDistributedSchedulingAMResponse dsRegResp = new RegisterDistributedSchedulingAMResponsePBImpl(dsProxy.registerApplicationMasterForDistributedScheduling(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(54321l, dsRegResp.getContainerIdStart());
    Assert.assertEquals(4, dsRegResp.getMaxContainerResource().getVirtualCores());
    Assert.assertEquals(1024, dsRegResp.getMinContainerResource().getMemorySize());
    Assert.assertEquals(2, dsRegResp.getIncrContainerResource().getVirtualCores());
    DistributedSchedulingAllocateRequestPBImpl distAllReq = (DistributedSchedulingAllocateRequestPBImpl) factory.newRecordInstance(DistributedSchedulingAllocateRequest.class);
    distAllReq.setAllocateRequest(allReq);
    distAllReq.setAllocatedContainers(Arrays.asList(c));
    DistributedSchedulingAllocateResponse dsAllocResp = new DistributedSchedulingAllocateResponsePBImpl(dsProxy.allocateForDistributedScheduling(null, distAllReq.getProto()));
    Assert.assertEquals("h1", dsAllocResp.getNodesForScheduling().get(0).getNodeId().getHost());
    FinishApplicationMasterResponse dsfinishResp = new FinishApplicationMasterResponsePBImpl(dsProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(false, dsfinishResp.getIsUnregistered());
}