Examples with LocalResourceVisibility org.apache.hadoop.yarn.api.records.LocalResourceVisibility used on opensource projects

Example 16 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class TestFSDownload method testDownload.

@Test(timeout = 10000)
public void testDownload() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int[] sizes = new int[10];
    for (int i = 0; i < 10; ++i) {
        sizes[i] = rand.nextInt(512) + 512;
        LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
        if (i % 2 == 1) {
            vis = LocalResourceVisibility.APPLICATION;
        }
        Path p = new Path(basedir, "" + i);
        LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);
        rsrcVis.put(rsrc, vis);
        Path destPath = dirs.getLocalPathForWrite(basedir.toString(), sizes[i], conf);
        destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
        FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
        pending.put(rsrc, exec.submit(fsd));
    }
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    for (Future<Path> path : pending.values()) {
        Assert.assertTrue(path.isDone());
    }
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            Path localized = p.getValue().get();
            assertEquals(sizes[Integer.parseInt(localized.getName())], p.getKey().getSize());
            FileStatus status = files.getFileStatus(localized.getParent());
            FsPermission perm = status.getPermission();
            assertEquals("Cache directory permissions are incorrect", new FsPermission((short) 0755), perm);
            status = files.getFileStatus(localized);
            perm = status.getPermission();
            System.out.println("File permission " + perm + " for rsrc vis " + p.getKey().getVisibility().name());
            assert (rsrcVis.containsKey(p.getKey()));
            Assert.assertTrue("Private file should be 500", perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
        }
    } catch (ExecutionException e) {
        throw new IOException("Failed exec", e);
    }
}

Example 17 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class TestFSDownload method downloadWithFileType.

private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    Path p = new Path(basedir, "" + 1);
    String strFileName = "";
    LocalResource rsrc = null;
    switch(fileType) {
        case TAR:
            rsrc = createTarFile(files, p, size, rand, vis);
            break;
        case JAR:
            rsrc = createJarFile(files, p, size, rand, vis);
            rsrc.setType(LocalResourceType.PATTERN);
            break;
        case ZIP:
            rsrc = createZipFile(files, p, size, rand, vis);
            strFileName = p.getName() + ".ZIP";
            break;
        case TGZ:
            rsrc = createTgzFile(files, p, size, rand, vis);
            break;
    }
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    pending.put(rsrc, exec.submit(fsd));
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    try {
        // see if there was an Exception during download
        pending.get(rsrc).get();
        FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus(basedir);
        for (FileStatus filestatus : filesstatus) {
            if (filestatus.isDirectory()) {
                FileStatus[] childFiles = files.getDefaultFileSystem().listStatus(filestatus.getPath());
                for (FileStatus childfile : childFiles) {
                    if (strFileName.endsWith(".ZIP") && childfile.getPath().getName().equals(strFileName) && !childfile.isDirectory()) {
                        Assert.fail("Failure...After unzip, there should have been a" + " directory formed with zip file name but found a file. " + childfile.getPath());
                    }
                    if (childfile.getPath().getName().startsWith("tmp")) {
                        Assert.fail("Tmp File should not have been there " + childfile.getPath());
                    }
                }
            }
        }
    } catch (Exception e) {
        throw new IOException("Failed exec", e);
    }
}

Example 18 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class ResourceLocalizationService method handleCleanupContainerResources.

@SuppressWarnings("unchecked")
private void handleCleanupContainerResources(ContainerLocalizationCleanupEvent rsrcCleanup) {
    Container c = rsrcCleanup.getContainer();
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcCleanup.getResources();
    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) {
        LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId().getApplicationId());
        for (LocalResourceRequest req : e.getValue()) {
            tracker.handle(new ResourceReleaseEvent(req, c.getContainerId()));
        }
    }
    String locId = c.getContainerId().toString();
    localizerTracker.cleanupPrivLocalizers(locId);
    // Delete the container directories
    String userName = c.getUser();
    String containerIDStr = c.toString();
    String appIDStr = c.getContainerId().getApplicationAttemptId().getApplicationId().toString();
    for (String localDir : dirsHandler.getLocalDirsForCleanup()) {
        // Delete the user-owned container-dir
        Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
        Path userdir = new Path(usersdir, userName);
        Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
        Path appDir = new Path(allAppsdir, appIDStr);
        Path containerDir = new Path(appDir, containerIDStr);
        submitDirForDeletion(userName, containerDir);
        // Delete the nmPrivate container-dir
        Path sysDir = new Path(localDir, NM_PRIVATE_DIR);
        Path appSysDir = new Path(sysDir, appIDStr);
        Path containerSysDir = new Path(appSysDir, containerIDStr);
        submitDirForDeletion(null, containerSysDir);
    }
    dispatcher.getEventHandler().handle(new ContainerEvent(c.getContainerId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
}

Example 19 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class ResourceLocalizationService method handleInitContainerResources.

/**
   * For each of the requested resources for a container, determines the
   * appropriate {@link LocalResourcesTracker} and forwards a 
   * {@link LocalResourceRequest} to that tracker.
   */
private void handleInitContainerResources(ContainerLocalizationRequestEvent rsrcReqs) {
    Container c = rsrcReqs.getContainer();
    EnumSet<ContainerState> set = EnumSet.of(ContainerState.LOCALIZING, ContainerState.RUNNING, ContainerState.REINITIALIZING);
    if (!set.contains(c.getContainerState())) {
        LOG.warn(c.getContainerId() + " is at " + c.getContainerState() + " state, do not localize resources.");
        return;
    }
    // create a loading cache for the file statuses
    LoadingCache<Path, Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
    LocalizerContext ctxt = new LocalizerContext(c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources();
    for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) {
        LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId().getApplicationId());
        for (LocalResourceRequest req : e.getValue()) {
            tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Localizing " + req.getPath() + " for container " + c.getContainerId());
            }
        }
    }
}

Example 20 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerHeartbeatWhenAppCleaningUp.

@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testLocalizerHeartbeatWhenAppCleaningUp() throws Exception {
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, lfs.makeQualified(new Path(basedir, 0 + "")).toString());
    // Start dispatcher.
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    dispatcher.register(ApplicationEventType.class, mock(EventHandler.class));
    dispatcher.register(ContainerEventType.class, mock(EventHandler.class));
    DummyExecutor exec = new DummyExecutor();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    // Start resource localization service.
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        // Init application resources.
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(1234567890L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        when(app.toString()).thenReturn(appId.toString());
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // Initialize localizer.
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 46, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getAppMockedResource(r);
        final LocalResource resource2 = getAppMockedResource(r);
        // Send localization requests for container.
        // 2 resources generated with APPLICATION visibility.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> appResourceList = Arrays.asList(req1, req2);
        rsrcs.put(LocalResourceVisibility.APPLICATION, appResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        dispatcher.await();
        // Wait for localization to begin.
        exec.waitForLocalizers(1);
        final String containerIdStr = c.getContainerId().toString();
        LocalizerRunner locRunnerForContainer = spyService.getLocalizerRunner(containerIdStr);
        // Heartbeats from container localizer
        LocalResourceStatus rsrcSuccess = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(containerIdStr);
        when(rsrcSuccess.getResource()).thenReturn(resource1);
        when(rsrcSuccess.getLocalSize()).thenReturn(4344L);
        when(rsrcSuccess.getLocalPath()).thenReturn(getPath("/some/path"));
        when(rsrcSuccess.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList());
        // First heartbeat which schedules first resource.
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to be LIVE in Heartbeat.", LocalizerAction.LIVE, response.getLocalizerAction());
        // Cleanup application.
        spyService.handle(new ContainerLocalizationCleanupEvent(c, rsrcs));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app));
        dispatcher.await();
        try {
            // Directly send heartbeat to introduce race as app is being cleaned up.
            locRunnerForContainer.processHeartbeat(Collections.singletonList(rsrcSuccess));
        } catch (Exception e) {
            fail("Exception should not have been thrown on processing heartbeat");
        }
        // Send another heartbeat.
        response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to DIE in Heartbeat.", LocalizerAction.DIE, response.getLocalizerAction());
        exec.setStopLocalization();
    } finally {
        spyService.stop();
        dispatcher.stop();
    }
}