use of org.apache.hadoop.yarn.api.records.URL in project hadoop by apache.
the class TaskAttemptImpl method createLocalResource.
/**
* Create a {@link LocalResource} record with all the given parameters.
*/
private static LocalResource createLocalResource(FileSystem fc, Path file, LocalResourceType type, LocalResourceVisibility visibility) throws IOException {
FileStatus fstat = fc.getFileStatus(file);
URL resourceURL = URL.fromPath(fc.resolvePath(fstat.getPath()));
long resourceSize = fstat.getLen();
long resourceModificationTime = fstat.getModificationTime();
return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime);
}
use of org.apache.hadoop.yarn.api.records.URL in project hadoop by apache.
the class TestConverterUtils method testConvertUrlWithNoPort.
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
Path expectedPath = new Path("hdfs://foo.com");
URL url = URL.fromPath(expectedPath);
Path actualPath = url.toPath();
assertEquals(expectedPath, actualPath);
}
use of org.apache.hadoop.yarn.api.records.URL in project hadoop by apache.
the class TestContainersMonitor method testContainerKillOnMemoryOverflow.
@Test
public void testContainerKillOnMemoryOverflow() throws IOException, InterruptedException, YarnException {
if (!ProcfsBasedProcessTree.isAvailable()) {
return;
}
containerManager.start();
File scriptFile = new File(tmpDir, "scriptFile.sh");
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile = new File(tmpDir, "start_file.txt").getAbsoluteFile();
// So that start file is readable by the
fileWriter.write("\numask 0");
// test.
fileWriter.write("\necho Hello World! > " + processStartFile);
fileWriter.write("\necho $$ >> " + processStartFile);
fileWriter.write("\nsleep 15");
fileWriter.close();
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
// ////// Construct the Container-id
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
URL resource_alpha = URL.fromPath(localFS.makeQualified(new Path(scriptFile.getAbsolutePath())));
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
rsrc_alpha.setResource(resource_alpha);
rsrc_alpha.setSize(-1);
rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
rsrc_alpha.setType(LocalResourceType.FILE);
rsrc_alpha.setTimestamp(scriptFile.lastModified());
String destinationFile = "dest_file";
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
Resource r = BuilderUtils.newResource(0, 0);
ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER, Priority.newInstance(0), 0);
Token containerToken = BuilderUtils.newContainerToken(context.getNodeId(), containerManager.getContext().getContainerTokenSecretManager().createPassword(containerIdentifier), containerIdentifier);
StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
containerManager.startContainers(allRequests);
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
Thread.sleep(1000);
LOG.info("Waiting for process start-file to be created");
}
Assert.assertTrue("ProcessStartFile doesn't exist!", processStartFile.exists());
// Now verify the contents of the file
BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
Assert.assertEquals("Hello World!", reader.readLine());
// Get the pid of the process
String pid = reader.readLine().trim();
// No more lines
Assert.assertEquals(null, reader.readLine());
BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE, 60);
List<ContainerId> containerIds = new ArrayList<ContainerId>();
containerIds.add(cId);
GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);
Assert.assertEquals(ContainerExitStatus.KILLED_EXCEEDED_VMEM, containerStatus.getExitStatus());
String expectedMsgPattern = "Container \\[pid=" + pid + ",containerID=" + cId + "\\] is running beyond virtual memory limits. Current usage: " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B physical memory used; " + "[0-9.]+ ?[KMGTPE]?B of [0-9.]+ ?[KMGTPE]?B virtual memory used. " + "Killing container.\nDump of the process-tree for " + cId + " :\n";
Pattern pat = Pattern.compile(expectedMsgPattern);
Assert.assertEquals("Expected message pattern is: " + expectedMsgPattern + "\n\nObserved message is: " + containerStatus.getDiagnostics(), true, pat.matcher(containerStatus.getDiagnostics()).find());
// Assert that the process is not alive anymore
Assert.assertFalse("Process is still alive!", exec.signalContainer(new ContainerSignalContext.Builder().setUser(user).setPid(pid).setSignal(Signal.NULL).build()));
}
use of org.apache.hadoop.yarn.api.records.URL in project hadoop by apache.
the class TestResourceLocalizationService method testDownloadingResourcesOnContainerKill.
@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testDownloadingResourcesOnContainerKill() throws Exception {
List<Path> localDirs = new ArrayList<Path>();
String[] sDirs = new String[1];
localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
sDirs[0] = localDirs.get(0).toString();
conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
DrainDispatcher dispatcher = new DrainDispatcher();
dispatcher.init(conf);
dispatcher.start();
EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
dispatcher.register(ApplicationEventType.class, applicationBus);
EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
dispatcher.register(ContainerEventType.class, containerBus);
DummyExecutor exec = new DummyExecutor();
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.init(conf);
DeletionService delServiceReal = new DeletionService(exec);
DeletionService delService = spy(delServiceReal);
delService.init(new Configuration());
delService.start();
ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
ResourceLocalizationService spyService = spy(rawService);
doReturn(mockServer).when(spyService).createServer();
doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE);
final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE);
final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR);
final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0]));
final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir);
doAnswer(new Answer<FileStatus>() {
@Override
public FileStatus answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
if (args.length > 0) {
if (args[0].equals(userDir) || args[0].equals(fileDir)) {
return fs;
}
}
return nmFs;
}
}).when(spylfs).getFileStatus(isA(Path.class));
try {
spyService.init(conf);
spyService.start();
final Application app = mock(Application.class);
final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
String user = "user0";
when(app.getUser()).thenReturn(user);
when(app.getAppId()).thenReturn(appId);
spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() {
@Override
public boolean matches(Object o) {
ApplicationEvent evt = (ApplicationEvent) o;
return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID();
}
};
dispatcher.await();
verify(applicationBus).handle(argThat(matchesAppInit));
// Initialize localizer.
Random r = new Random();
long seed = r.nextLong();
System.out.println("SEED: " + seed);
r.setSeed(seed);
final Container c1 = getMockContainer(appId, 42, "user0");
final Container c2 = getMockContainer(appId, 43, "user0");
FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
final LocalResource resource1 = getPrivateMockedResource(r);
LocalResource resource2 = null;
do {
resource2 = getPrivateMockedResource(r);
} while (resource2 == null || resource2.equals(resource1));
LocalResource resource3 = null;
do {
resource3 = getPrivateMockedResource(r);
} while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2));
// Send localization requests for container c1 and c2.
final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
privateResourceList.add(req1);
privateResourceList.add(req2);
privateResourceList.add(req3);
rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs));
final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2);
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
List<LocalResourceRequest> privateResourceList1 = new ArrayList<LocalResourceRequest>();
privateResourceList1.add(req1_1);
rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1);
spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1));
dispatcher.await();
// Wait for localizers of both container c1 and c2 to begin.
exec.waitForLocalizers(2);
LocalizerRunner locC1 = spyService.getLocalizerRunner(c1.getContainerId().toString());
final String containerIdStr = c1.getContainerId().toString();
// Heartbeats from container localizer
LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
LocalizerStatus stat = mock(LocalizerStatus.class);
when(stat.getLocalizerId()).thenReturn(containerIdStr);
when(rsrc1success.getResource()).thenReturn(resource1);
when(rsrc2pending.getResource()).thenReturn(resource2);
when(rsrc1success.getLocalSize()).thenReturn(4344L);
URL locPath = getPath("/some/path");
when(rsrc1success.getLocalPath()).thenReturn(locPath);
when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING);
when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()).thenReturn(Collections.singletonList(rsrc1success)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(Collections.<LocalResourceStatus>emptyList());
// First heartbeat which schedules first resource.
LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
// Second heartbeat which reports first resource as success.
// Second resource is scheduled.
response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
final String locPath1 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile();
// Third heartbeat which reports second resource as pending.
// Third resource is scheduled.
response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
final String locPath2 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile();
// Container c1 is killed which leads to cleanup
spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs));
// This heartbeat will indicate to container localizer to die as localizer
// runner has stopped.
response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
exec.setStopLocalization();
dispatcher.await();
// verify container notification
ArgumentMatcher<ContainerEvent> successContainerLoc = new ArgumentMatcher<ContainerEvent>() {
@Override
public boolean matches(Object o) {
ContainerEvent evt = (ContainerEvent) o;
return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c1.getContainerId() == evt.getContainerID();
}
};
// Only one resource gets localized for container c1.
verify(containerBus).handle(argThat(successContainerLoc));
Set<Path> paths = Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), new Path(locPath2), new Path(locPath2 + "_tmp"));
// Wait for localizer runner thread for container c1 to finish.
while (locC1.getState() != Thread.State.TERMINATED) {
Thread.sleep(50);
}
// Verify if downloading resources were submitted for deletion.
verify(delService).delete(eq(user), (Path) eq(null), argThat(new DownloadingPathsMatcher(paths)));
LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, "user0", appId);
// Container c1 was killed but this resource was localized before kill
// hence its not removed despite ref cnt being 0.
LocalizedResource rsrc1 = tracker.getLocalizedResource(req1);
assertNotNull(rsrc1);
assertEquals(rsrc1.getState(), ResourceState.LOCALIZED);
assertEquals(rsrc1.getRefCount(), 0);
// Container c1 was killed but this resource is referenced by container c2
// as well hence its ref cnt is 1.
LocalizedResource rsrc2 = tracker.getLocalizedResource(req2);
assertNotNull(rsrc2);
assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING);
assertEquals(rsrc2.getRefCount(), 1);
// As container c1 was killed and this resource was not referenced by any
// other container, hence its removed.
LocalizedResource rsrc3 = tracker.getLocalizedResource(req3);
assertNull(rsrc3);
} finally {
spyService.stop();
dispatcher.stop();
delService.stop();
}
}
use of org.apache.hadoop.yarn.api.records.URL in project hadoop by apache.
the class TestResourceLocalizationService method testPublicResourceAddResourceExceptions.
/*
* Test case for handling RejectedExecutionException and IOException which can
* be thrown when adding public resources to the pending queue.
* RejectedExecutionException can be thrown either due to the incoming queue
* being full or if the ExecutorCompletionService threadpool is shutdown.
* Since it's hard to simulate the queue being full, this test just shuts down
* the threadpool and makes sure the exception is handled. If anything is
* messed up the async dispatcher thread will cause a system exit causing the
* test to fail.
*/
@Test
@SuppressWarnings("unchecked")
public void testPublicResourceAddResourceExceptions() throws Exception {
List<Path> localDirs = new ArrayList<Path>();
String[] sDirs = new String[4];
for (int i = 0; i < 4; ++i) {
localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
sDirs[i] = localDirs.get(i).toString();
}
conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
DrainDispatcher dispatcher = new DrainDispatcher();
EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
dispatcher.register(ApplicationEventType.class, applicationBus);
EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
dispatcher.register(ContainerEventType.class, containerBus);
ContainerExecutor exec = mock(ContainerExecutor.class);
DeletionService delService = mock(DeletionService.class);
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
dirsHandlerSpy.init(conf);
dispatcher.init(conf);
dispatcher.start();
try {
ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
ResourceLocalizationService spyService = spy(rawService);
doReturn(mockServer).when(spyService).createServer();
doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
spyService.init(conf);
spyService.start();
final String user = "user0";
// init application
final Application app = mock(Application.class);
final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
when(app.getUser()).thenReturn(user);
when(app.getAppId()).thenReturn(appId);
spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
dispatcher.await();
// init resources
Random r = new Random();
r.setSeed(r.nextLong());
// Queue localization request for the public resource
final LocalResource pubResource = getPublicMockedResource(r);
final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
// init container.
final Container c = getMockContainer(appId, 42, user);
// first test ioexception
Mockito.doThrow(new IOException()).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean());
// send request
spyService.handle(new ContainerLocalizationRequestEvent(c, req));
dispatcher.await();
LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
Assert.assertNull(tracker.getLocalizedResource(pubReq));
// test IllegalArgumentException
String name = Long.toHexString(r.nextLong());
URL url = getPath("/local/PRIVATE/" + name + "/");
final LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc);
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
Mockito.doCallRealMethod().when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean());
// send request
spyService.handle(new ContainerLocalizationRequestEvent(c, req1));
dispatcher.await();
tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
Assert.assertNull(tracker.getLocalizedResource(pubReq));
// test RejectedExecutionException by shutting down the thread pool
PublicLocalizer publicLocalizer = spyService.getPublicLocalizer();
publicLocalizer.threadPool.shutdown();
spyService.handle(new ContainerLocalizationRequestEvent(c, req));
dispatcher.await();
tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
Assert.assertNull(tracker.getLocalizedResource(pubReq));
} finally {
// if we call stop with events in the queue, an InterruptedException gets
// thrown resulting in the dispatcher thread causing a system exit
dispatcher.await();
dispatcher.stop();
}
}
Aggregations