Search in sources :

Example 1 with ApplicationMasterProtocolPB

use of org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB in project hadoop by apache.

the class TestOpportunisticContainerAllocatorAMService method testRPCWrapping.

// Test if the OpportunisticContainerAllocatorAMService can handle both
// DSProtocol as well as AMProtocol clients
@Test
public void testRPCWrapping() throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, addr);
    final RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
    final RMContext rmContext = new RMContextImpl() {

        @Override
        public AMLivelinessMonitor getAMLivelinessMonitor() {
            return null;
        }

        @Override
        public Configuration getYarnConfiguration() {
            return new YarnConfiguration();
        }

        @Override
        public RMContainerTokenSecretManager getContainerTokenSecretManager() {
            return new RMContainerTokenSecretManager(conf);
        }
    };
    Container c = factory.newRecordInstance(Container.class);
    c.setExecutionType(ExecutionType.OPPORTUNISTIC);
    c.setId(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(12345, 1), 2), 3));
    AllocateRequest allReq = (AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class);
    allReq.setAskList(Arrays.asList(ResourceRequest.newInstance(Priority.UNDEFINED, "a", Resource.newInstance(1, 2), 1, true, "exp", ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true))));
    OpportunisticContainerAllocatorAMService service = createService(factory, rmContext, c);
    conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
    Server server = service.getServer(rpc, conf, addr, null);
    server.start();
    // Verify that the OpportunisticContainerAllocatorAMSercvice can handle
    // vanilla ApplicationMasterProtocol clients
    RPC.setProtocolEngine(conf, ApplicationMasterProtocolPB.class, ProtobufRpcEngine.class);
    ApplicationMasterProtocolPB ampProxy = RPC.getProxy(ApplicationMasterProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
    RegisterApplicationMasterResponse regResp = new RegisterApplicationMasterResponsePBImpl(ampProxy.registerApplicationMaster(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals("dummyQueue", regResp.getQueue());
    FinishApplicationMasterResponse finishResp = new FinishApplicationMasterResponsePBImpl(ampProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(false, finishResp.getIsUnregistered());
    AllocateResponse allocResp = new AllocateResponsePBImpl(ampProxy.allocate(null, ((AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class)).getProto()));
    List<Container> allocatedContainers = allocResp.getAllocatedContainers();
    Assert.assertEquals(1, allocatedContainers.size());
    Assert.assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainers.get(0).getExecutionType());
    Assert.assertEquals(12345, allocResp.getNumClusterNodes());
    // Verify that the DistrubutedSchedulingService can handle the
    // DistributedSchedulingAMProtocol clients as well
    RPC.setProtocolEngine(conf, DistributedSchedulingAMProtocolPB.class, ProtobufRpcEngine.class);
    DistributedSchedulingAMProtocolPB dsProxy = RPC.getProxy(DistributedSchedulingAMProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
    RegisterDistributedSchedulingAMResponse dsRegResp = new RegisterDistributedSchedulingAMResponsePBImpl(dsProxy.registerApplicationMasterForDistributedScheduling(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(54321l, dsRegResp.getContainerIdStart());
    Assert.assertEquals(4, dsRegResp.getMaxContainerResource().getVirtualCores());
    Assert.assertEquals(1024, dsRegResp.getMinContainerResource().getMemorySize());
    Assert.assertEquals(2, dsRegResp.getIncrContainerResource().getVirtualCores());
    DistributedSchedulingAllocateRequestPBImpl distAllReq = (DistributedSchedulingAllocateRequestPBImpl) factory.newRecordInstance(DistributedSchedulingAllocateRequest.class);
    distAllReq.setAllocateRequest(allReq);
    distAllReq.setAllocatedContainers(Arrays.asList(c));
    DistributedSchedulingAllocateResponse dsAllocResp = new DistributedSchedulingAllocateResponsePBImpl(dsProxy.allocateForDistributedScheduling(null, distAllReq.getProto()));
    Assert.assertEquals("h1", dsAllocResp.getNodesForScheduling().get(0).getNodeId().getHost());
    FinishApplicationMasterResponse dsfinishResp = new FinishApplicationMasterResponsePBImpl(dsProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
    Assert.assertEquals(false, dsfinishResp.getIsUnregistered());
}
Also used : AllocateResponsePBImpl(org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl) DistributedSchedulingAllocateResponsePBImpl(org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateResponsePBImpl) CapacitySchedulerConfiguration(org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Server(org.apache.hadoop.ipc.Server) InetSocketAddress(java.net.InetSocketAddress) DistributedSchedulingAllocateRequest(org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) FinishApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse) RegisterDistributedSchedulingAMResponsePBImpl(org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistributedSchedulingAMResponsePBImpl) HadoopYarnProtoRPC(org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC) FinishApplicationMasterResponsePBImpl(org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterResponsePBImpl) ApplicationMasterProtocolPB(org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB) FinishApplicationMasterRequest(org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest) DistributedSchedulingAllocateResponse(org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) AllocateRequestPBImpl(org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl) DistributedSchedulingAllocateRequestPBImpl(org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateRequestPBImpl) RMContainer(org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer) Container(org.apache.hadoop.yarn.api.records.Container) FinishApplicationMasterRequestPBImpl(org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterRequestPBImpl) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) RegisterApplicationMasterResponsePBImpl(org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl) RMContainerTokenSecretManager(org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager) RegisterApplicationMasterRequest(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest) DistributedSchedulingAllocateResponsePBImpl(org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateResponsePBImpl) RegisterDistributedSchedulingAMResponse(org.apache.hadoop.yarn.server.api.protocolrecords.RegisterDistributedSchedulingAMResponse) RegisterApplicationMasterRequestPBImpl(org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl) YarnRPC(org.apache.hadoop.yarn.ipc.YarnRPC) DistributedSchedulingAllocateResponse(org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse) RecordFactory(org.apache.hadoop.yarn.factories.RecordFactory) RegisterApplicationMasterResponse(org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse) DistributedSchedulingAllocateRequestPBImpl(org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateRequestPBImpl) DistributedSchedulingAMProtocolPB(org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB) DistributedSchedulingAllocateRequest(org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest) Test(org.junit.Test)

Aggregations

InetSocketAddress (java.net.InetSocketAddress)1 Configuration (org.apache.hadoop.conf.Configuration)1 Server (org.apache.hadoop.ipc.Server)1 ApplicationMasterProtocolPB (org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB)1 AllocateRequest (org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)1 AllocateResponse (org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse)1 FinishApplicationMasterRequest (org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest)1 FinishApplicationMasterResponse (org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse)1 RegisterApplicationMasterRequest (org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest)1 RegisterApplicationMasterResponse (org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse)1 AllocateRequestPBImpl (org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl)1 AllocateResponsePBImpl (org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl)1 FinishApplicationMasterRequestPBImpl (org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterRequestPBImpl)1 FinishApplicationMasterResponsePBImpl (org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterResponsePBImpl)1 RegisterApplicationMasterRequestPBImpl (org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl)1 RegisterApplicationMasterResponsePBImpl (org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl)1 Container (org.apache.hadoop.yarn.api.records.Container)1 YarnConfiguration (org.apache.hadoop.yarn.conf.YarnConfiguration)1 RecordFactory (org.apache.hadoop.yarn.factories.RecordFactory)1 HadoopYarnProtoRPC (org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC)1