use of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse in project hadoop by apache.
the class RMCommunicator method doUnregistration.
@VisibleForTesting
protected void doUnregistration() throws YarnException, IOException, InterruptedException {
FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
JobImpl jobImpl = (JobImpl) job;
if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
finishState = FinalApplicationStatus.SUCCEEDED;
} else if (jobImpl.getInternalState() == JobStateInternal.KILLED || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
finishState = FinalApplicationStatus.KILLED;
} else if (jobImpl.getInternalState() == JobStateInternal.FAILED || jobImpl.getInternalState() == JobStateInternal.ERROR) {
finishState = FinalApplicationStatus.FAILED;
}
StringBuffer sb = new StringBuffer();
for (String s : job.getDiagnostics()) {
sb.append(s).append("\n");
}
LOG.info("Setting job diagnostics to " + sb.toString());
String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(), context.getApplicationID());
LOG.info("History url is " + historyUrl);
FinishApplicationMasterRequest request = FinishApplicationMasterRequest.newInstance(finishState, sb.toString(), historyUrl);
try {
while (true) {
FinishApplicationMasterResponse response = scheduler.finishApplicationMaster(request);
if (response.getIsUnregistered()) {
// When excepting ClientService, other services are already stopped,
// it is safe to let clients know the final states. ClientService
// should wait for some time so clients have enough time to know the
// final states.
RunningAppContext raContext = (RunningAppContext) context;
raContext.markSuccessfulUnregistration();
break;
}
LOG.info("Waiting for application to be successfully unregistered.");
Thread.sleep(rmPollInterval);
}
} catch (ApplicationMasterNotRegisteredException e) {
// RM might have restarted or failed over and so lost the fact that AM had
// registered before.
register();
doUnregistration();
}
}
use of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse in project hadoop by apache.
the class TestOpportunisticContainerAllocatorAMService method testRPCWrapping.
// Test if the OpportunisticContainerAllocatorAMService can handle both
// DSProtocol as well as AMProtocol clients
@Test
public void testRPCWrapping() throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, addr);
final RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
final RMContext rmContext = new RMContextImpl() {
@Override
public AMLivelinessMonitor getAMLivelinessMonitor() {
return null;
}
@Override
public Configuration getYarnConfiguration() {
return new YarnConfiguration();
}
@Override
public RMContainerTokenSecretManager getContainerTokenSecretManager() {
return new RMContainerTokenSecretManager(conf);
}
};
Container c = factory.newRecordInstance(Container.class);
c.setExecutionType(ExecutionType.OPPORTUNISTIC);
c.setId(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(12345, 1), 2), 3));
AllocateRequest allReq = (AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class);
allReq.setAskList(Arrays.asList(ResourceRequest.newInstance(Priority.UNDEFINED, "a", Resource.newInstance(1, 2), 1, true, "exp", ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true))));
OpportunisticContainerAllocatorAMService service = createService(factory, rmContext, c);
conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
Server server = service.getServer(rpc, conf, addr, null);
server.start();
// Verify that the OpportunisticContainerAllocatorAMSercvice can handle
// vanilla ApplicationMasterProtocol clients
RPC.setProtocolEngine(conf, ApplicationMasterProtocolPB.class, ProtobufRpcEngine.class);
ApplicationMasterProtocolPB ampProxy = RPC.getProxy(ApplicationMasterProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
RegisterApplicationMasterResponse regResp = new RegisterApplicationMasterResponsePBImpl(ampProxy.registerApplicationMaster(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
Assert.assertEquals("dummyQueue", regResp.getQueue());
FinishApplicationMasterResponse finishResp = new FinishApplicationMasterResponsePBImpl(ampProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
Assert.assertEquals(false, finishResp.getIsUnregistered());
AllocateResponse allocResp = new AllocateResponsePBImpl(ampProxy.allocate(null, ((AllocateRequestPBImpl) factory.newRecordInstance(AllocateRequest.class)).getProto()));
List<Container> allocatedContainers = allocResp.getAllocatedContainers();
Assert.assertEquals(1, allocatedContainers.size());
Assert.assertEquals(ExecutionType.OPPORTUNISTIC, allocatedContainers.get(0).getExecutionType());
Assert.assertEquals(12345, allocResp.getNumClusterNodes());
// Verify that the DistrubutedSchedulingService can handle the
// DistributedSchedulingAMProtocol clients as well
RPC.setProtocolEngine(conf, DistributedSchedulingAMProtocolPB.class, ProtobufRpcEngine.class);
DistributedSchedulingAMProtocolPB dsProxy = RPC.getProxy(DistributedSchedulingAMProtocolPB.class, 1, NetUtils.getConnectAddress(server), conf);
RegisterDistributedSchedulingAMResponse dsRegResp = new RegisterDistributedSchedulingAMResponsePBImpl(dsProxy.registerApplicationMasterForDistributedScheduling(null, ((RegisterApplicationMasterRequestPBImpl) factory.newRecordInstance(RegisterApplicationMasterRequest.class)).getProto()));
Assert.assertEquals(54321l, dsRegResp.getContainerIdStart());
Assert.assertEquals(4, dsRegResp.getMaxContainerResource().getVirtualCores());
Assert.assertEquals(1024, dsRegResp.getMinContainerResource().getMemorySize());
Assert.assertEquals(2, dsRegResp.getIncrContainerResource().getVirtualCores());
DistributedSchedulingAllocateRequestPBImpl distAllReq = (DistributedSchedulingAllocateRequestPBImpl) factory.newRecordInstance(DistributedSchedulingAllocateRequest.class);
distAllReq.setAllocateRequest(allReq);
distAllReq.setAllocatedContainers(Arrays.asList(c));
DistributedSchedulingAllocateResponse dsAllocResp = new DistributedSchedulingAllocateResponsePBImpl(dsProxy.allocateForDistributedScheduling(null, distAllReq.getProto()));
Assert.assertEquals("h1", dsAllocResp.getNodesForScheduling().get(0).getNodeId().getHost());
FinishApplicationMasterResponse dsfinishResp = new FinishApplicationMasterResponsePBImpl(dsProxy.finishApplicationMaster(null, ((FinishApplicationMasterRequestPBImpl) factory.newRecordInstance(FinishApplicationMasterRequest.class)).getProto()));
Assert.assertEquals(false, dsfinishResp.getIsUnregistered());
}
use of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse in project hadoop by apache.
the class TestAMRMProxyService method testFinishOneApplicationMasterWithFailure.
@Test
public void testFinishOneApplicationMasterWithFailure() throws Exception {
int testAppId = 1;
RegisterApplicationMasterResponse registerResponse = registerApplicationMaster(testAppId);
Assert.assertNotNull(registerResponse);
Assert.assertEquals(Integer.toString(testAppId), registerResponse.getQueue());
FinishApplicationMasterResponse finshResponse = finishApplicationMaster(testAppId, FinalApplicationStatus.FAILED);
Assert.assertNotNull(finshResponse);
Assert.assertEquals(false, finshResponse.getIsUnregistered());
try {
// Try to finish an application master that is already finished.
finishApplicationMaster(testAppId, FinalApplicationStatus.SUCCEEDED);
Assert.fail("The request to finish application master should have failed");
} catch (Throwable ex) {
// This is expected. So nothing required here.
LOG.info("Finish registration failed as expected because it was not registered");
}
}
use of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse in project hadoop by apache.
the class TestAMRMProxyService method testAllocateRequestWithNullValues.
@Test
public void testAllocateRequestWithNullValues() throws Exception {
int testAppId = 1;
RegisterApplicationMasterResponse registerResponse = registerApplicationMaster(testAppId);
Assert.assertNotNull(registerResponse);
Assert.assertEquals(Integer.toString(testAppId), registerResponse.getQueue());
AllocateResponse allocateResponse = allocate(testAppId);
Assert.assertNotNull(allocateResponse);
FinishApplicationMasterResponse finshResponse = finishApplicationMaster(testAppId, FinalApplicationStatus.SUCCEEDED);
Assert.assertNotNull(finshResponse);
Assert.assertEquals(true, finshResponse.getIsUnregistered());
}
use of org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse in project hadoop by apache.
the class TestAMRMProxyService method testFinishMulitpleApplicationMasters.
@Test
public void testFinishMulitpleApplicationMasters() throws Exception {
int numberOfRequests = 3;
for (int index = 0; index < numberOfRequests; index++) {
RegisterApplicationMasterResponse registerResponse = registerApplicationMaster(index);
Assert.assertNotNull(registerResponse);
Assert.assertEquals(Integer.toString(index), registerResponse.getQueue());
}
// Finish in reverse sequence
for (int index = numberOfRequests - 1; index >= 0; index--) {
FinishApplicationMasterResponse finshResponse = finishApplicationMaster(index, FinalApplicationStatus.SUCCEEDED);
Assert.assertNotNull(finshResponse);
Assert.assertEquals(true, finshResponse.getIsUnregistered());
// Assert that the application has been removed from the collection
Assert.assertTrue(this.getAMRMProxyService().getPipelines().size() == index);
}
try {
// Try to finish an application master that is already finished.
finishApplicationMaster(1, FinalApplicationStatus.SUCCEEDED);
Assert.fail("The request to finish application master should have failed");
} catch (Throwable ex) {
// This is expected. So nothing required here.
LOG.info("Finish registration failed as expected because it was not registered");
}
try {
// Try to finish an application master that was not registered.
finishApplicationMaster(4, FinalApplicationStatus.SUCCEEDED);
Assert.fail("The request to finish application master should have failed");
} catch (Throwable ex) {
// This is expected. So nothing required here.
LOG.info("Finish registration failed as expected because it was not registered");
}
}
Aggregations