use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestContainerManager method testNullTokens.
/* Test added to verify fix in YARN-644 */
@Test
public void testNullTokens() throws Exception {
ContainerManagerImpl cMgrImpl = new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater, metrics, dirsHandler);
String strExceptionMsg = "";
try {
cMgrImpl.authorizeStartAndResourceIncreaseRequest(null, new ContainerTokenIdentifier(), true);
} catch (YarnException ye) {
strExceptionMsg = ye.getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
strExceptionMsg = "";
try {
cMgrImpl.authorizeStartAndResourceIncreaseRequest(new NMTokenIdentifier(), null, true);
} catch (YarnException ye) {
strExceptionMsg = ye.getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_CONTAINERTOKEN_MSG);
strExceptionMsg = "";
try {
cMgrImpl.authorizeGetAndStopContainerRequest(null, null, true, null);
} catch (YarnException ye) {
strExceptionMsg = ye.getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
strExceptionMsg = "";
try {
cMgrImpl.authorizeUser(null, null);
} catch (YarnException ye) {
strExceptionMsg = ye.getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
ContainerManagerImpl spyContainerMgr = Mockito.spy(cMgrImpl);
UserGroupInformation ugInfo = UserGroupInformation.createRemoteUser("a");
Mockito.when(spyContainerMgr.getRemoteUgi()).thenReturn(ugInfo);
Mockito.when(spyContainerMgr.selectNMTokenIdentifier(ugInfo)).thenReturn(null);
strExceptionMsg = "";
try {
spyContainerMgr.stopContainers(new StopContainersRequestPBImpl());
} catch (YarnException ye) {
strExceptionMsg = ye.getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
strExceptionMsg = "";
try {
spyContainerMgr.getContainerStatuses(new GetContainerStatusesRequestPBImpl());
} catch (YarnException ye) {
strExceptionMsg = ye.getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_NMTOKEN_MSG);
Mockito.doNothing().when(spyContainerMgr).authorizeUser(ugInfo, null);
List<StartContainerRequest> reqList = new ArrayList<>();
reqList.add(StartContainerRequest.newInstance(null, null));
StartContainersRequest reqs = new StartContainersRequestPBImpl();
reqs.setStartContainerRequests(reqList);
strExceptionMsg = "";
try {
spyContainerMgr.startContainers(reqs);
} catch (YarnException ye) {
strExceptionMsg = ye.getCause().getMessage();
}
Assert.assertEquals(strExceptionMsg, ContainerManagerImpl.INVALID_CONTAINERTOKEN_MSG);
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestNodeManagerShutdown method startContainer.
public static void startContainer(NodeManager nm, ContainerId cId, FileContext localFS, File scriptFileDir, File processStartFile, final int port) throws IOException, YarnException {
File scriptFile = createUnhaltingScriptFile(cId, scriptFileDir, processStartFile);
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
NodeId nodeId = BuilderUtils.newNodeId(InetAddress.getByName("localhost").getCanonicalHostName(), port);
URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
LocalResource localResource = recordFactory.newRecordInstance(LocalResource.class);
localResource.setResource(localResourceUri);
localResource.setSize(-1);
localResource.setVisibility(LocalResourceVisibility.APPLICATION);
localResource.setType(LocalResourceType.FILE);
localResource.setTimestamp(scriptFile.lastModified());
String destinationFile = "dest_file";
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
localResources.put(destinationFile, localResource);
containerLaunchContext.setLocalResources(localResources);
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.toString());
org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken = ConverterUtils.convertFromYarn(nm.getNMContext().getNMTokenSecretManager().createNMToken(cId.getApplicationAttemptId(), nodeId, user), containerManagerBindAddress);
currentUser.addToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
@Override
public ContainerManagementProtocol run() {
Configuration conf = new Configuration();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", port);
return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, containerManagerBindAddress, conf);
}
});
StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0);
Assert.assertTrue(EnumSet.of(ContainerState.RUNNING, ContainerState.SCHEDULED).contains(containerStatus.getState()));
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestNMProxy method testShouldNotRetryForeverForNonNetworkExceptionsOnNMConnections.
@Test(timeout = 20000, expected = IOException.class)
public void testShouldNotRetryForeverForNonNetworkExceptionsOnNMConnections() throws Exception {
conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, -1);
StartContainersRequest allRequests = Records.newRecord(StartContainersRequest.class);
ContainerManagementProtocol proxy = getNMProxy(conf);
shouldThrowNMNotYetReadyException = false;
retryCount = 0;
proxy.startContainers(allRequests);
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestNMProxy method testNMProxyRPCRetry.
@Test(timeout = 20000)
public void testNMProxyRPCRetry() throws Exception {
conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 1000);
conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
StartContainersRequest allRequests = Records.newRecord(StartContainersRequest.class);
Configuration newConf = new YarnConfiguration(conf);
newConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 100);
newConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100);
// connect to some dummy address so that it can trigger
// connection failure and RPC level retires.
newConf.set(YarnConfiguration.NM_ADDRESS, "1234");
ContainerManagementProtocol proxy = getNMProxy(newConf);
try {
proxy.startContainers(allRequests);
Assert.fail("should get socket exception");
} catch (IOException e) {
// socket exception should be thrown immediately, without RPC retries.
Assert.assertTrue(e instanceof java.net.SocketException);
}
}
use of org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest in project hadoop by apache.
the class TestNodeManagerReboot method testClearLocalDirWhenNodeReboot.
@Test(timeout = 2000000)
public void testClearLocalDirWhenNodeReboot() throws IOException, YarnException, InterruptedException {
nm = new MyNodeManager();
nm.start();
final ContainerManagementProtocol containerManager = nm.getContainerManager();
// create files under fileCache
createFiles(nmLocalDir.getAbsolutePath(), ContainerLocalizer.FILECACHE, 100);
localResourceDir.mkdirs();
ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
// Construct the Container-id
ContainerId cId = createContainerId();
URL localResourceUri = URL.fromPath(localFS.makeQualified(new Path(localResourceDir.getAbsolutePath())));
LocalResource localResource = LocalResource.newInstance(localResourceUri, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, -1, localResourceDir.lastModified());
String destinationFile = "dest_file";
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
localResources.put(destinationFile, localResource);
containerLaunchContext.setLocalResources(localResources);
List<String> commands = new ArrayList<String>();
containerLaunchContext.setCommands(commands);
NodeId nodeId = nm.getNMContext().getNodeId();
StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, TestContainerManager.createContainerToken(cId, 0, nodeId, destinationFile, nm.getNMContext().getContainerTokenSecretManager()));
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
final StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
final UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.getApplicationAttemptId().toString());
NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.getApplicationAttemptId(), nodeId, user, 123);
currentUser.addTokenIdentifier(nmIdentifier);
currentUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws YarnException, IOException {
nm.getContainerManager().startContainers(allRequests);
return null;
}
});
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
Container container = nm.getNMContext().getContainers().get(request.getContainerIds().get(0));
final int MAX_TRIES = 20;
int numTries = 0;
while (!container.getContainerState().equals(ContainerState.DONE) && numTries <= MAX_TRIES) {
try {
Thread.sleep(500);
} catch (InterruptedException ex) {
// Do nothing
}
numTries++;
}
Assert.assertEquals(ContainerState.DONE, container.getContainerState());
Assert.assertTrue("The container should create a subDir named currentUser: " + user + "under localDir/usercache", numOfLocalDirs(nmLocalDir.getAbsolutePath(), ContainerLocalizer.USERCACHE) > 0);
Assert.assertTrue("There should be files or Dirs under nm_private when " + "container is launched", numOfLocalDirs(nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0);
// restart the NodeManager
restartNM(MAX_TRIES);
checkNumOfLocalDirs();
verify(delService, times(1)).delete((String) isNull(), argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR + "_DEL_")));
verify(delService, times(1)).delete((String) isNull(), argThat(new PathInclude(ContainerLocalizer.FILECACHE + "_DEL_")));
verify(delService, times(1)).scheduleFileDeletionTask(argThat(new FileDeletionInclude(user, null, new String[] { destinationFile })));
verify(delService, times(1)).scheduleFileDeletionTask(argThat(new FileDeletionInclude(null, ContainerLocalizer.USERCACHE + "_DEL_", new String[] {})));
// restart the NodeManager again
// this time usercache directory should be empty
restartNM(MAX_TRIES);
checkNumOfLocalDirs();
}
Aggregations