Examples with LocalResourceVisibility org.apache.hadoop.yarn.api.records.LocalResourceVisibility used on opensource projects

Example 1 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class TestFSDownload method testUniqueDestinationPath.

@Test(timeout = 10000)
public void testUniqueDestinationPath() throws Exception {
    Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    ExecutorService singleThreadedExec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    Path p = new Path(basedir, "dir" + 0 + ".jar");
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    LocalResource rsrc = createJar(files, p, vis);
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    Future<Path> rPath = singleThreadedExec.submit(fsd);
    singleThreadedExec.shutdown();
    while (!singleThreadedExec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    Assert.assertTrue(rPath.isDone());
    // Now FSDownload will not create a random directory to localize the
    // resource. Therefore the final localizedPath for the resource should be
    // destination directory (passed as an argument) + file name.
    Assert.assertEquals(destPath, rPath.get().getParent());
}

Example 2 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class TestFSDownload method testDownloadBadPublic.

@Test(timeout = 10000)
public void testDownloadBadPublic() throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int size = 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
    Path path = new Path(basedir, "test-file");
    LocalResource rsrc = createFile(files, path, size, rand, vis);
    rsrcVis.put(rsrc, vis);
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    pending.put(rsrc, exec.submit(fsd));
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    Assert.assertTrue(pending.get(rsrc).isDone());
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            p.getValue().get();
            Assert.fail("We localized a file that is not public.");
        }
    } catch (ExecutionException e) {
        Assert.assertTrue(e.getCause() instanceof IOException);
    }
}

Example 3 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class TestFSDownload method testDirDownload.

@Test(timeout = 10000)
public void testDirDownload() throws IOException, InterruptedException {
    Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Map<LocalResource, LocalResourceVisibility> rsrcVis = new HashMap<LocalResource, LocalResourceVisibility>();
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    for (int i = 0; i < 5; ++i) {
        LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
        if (i % 2 == 1) {
            vis = LocalResourceVisibility.APPLICATION;
        }
        Path p = new Path(basedir, "dir" + i + ".jar");
        LocalResource rsrc = createJar(files, p, vis);
        rsrcVis.put(rsrc, vis);
        Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
        destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
        FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
        pending.put(rsrc, exec.submit(fsd));
    }
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    for (Future<Path> path : pending.values()) {
        Assert.assertTrue(path.isDone());
    }
    try {
        for (Map.Entry<LocalResource, Future<Path>> p : pending.entrySet()) {
            Path localized = p.getValue().get();
            FileStatus status = files.getFileStatus(localized);
            System.out.println("Testing path " + localized);
            assert (status.isDirectory());
            assert (rsrcVis.containsKey(p.getKey()));
            verifyPermsRecursively(localized.getFileSystem(conf), files, localized, rsrcVis.get(p.getKey()));
        }
    } catch (ExecutionException e) {
        throw new IOException("Failed exec", e);
    }
}

Example 4 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class ContainerManagerImpl method localize.

@Override
@SuppressWarnings("unchecked")
public ResourceLocalizationResponse localize(ResourceLocalizationRequest request) throws YarnException, IOException {
    ContainerId containerId = request.getContainerId();
    Container container = preReInitializeOrLocalizeCheck(containerId, ReInitOp.LOCALIZE);
    try {
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = container.getResourceSet().addResources(request.getLocalResources());
        if (req != null && !req.isEmpty()) {
            dispatcher.getEventHandler().handle(new ContainerLocalizationRequestEvent(container, req));
        }
    } catch (URISyntaxException e) {
        LOG.info("Error when parsing local resource URI for " + containerId, e);
        throw new YarnException(e);
    }
    return ResourceLocalizationResponse.newInstance();
}

Example 5 with LocalResourceVisibility

use of org.apache.hadoop.yarn.api.records.LocalResourceVisibility in project hadoop by apache.

the class ResourceSet method addResources.

public Map<LocalResourceVisibility, Collection<LocalResourceRequest>> addResources(Map<String, LocalResource> localResourceMap) throws URISyntaxException {
    if (localResourceMap == null || localResourceMap.isEmpty()) {
        return null;
    }
    Map<LocalResourceRequest, Set<String>> allResources = new HashMap<>();
    List<LocalResourceRequest> publicList = new ArrayList<>();
    List<LocalResourceRequest> privateList = new ArrayList<>();
    List<LocalResourceRequest> appList = new ArrayList<>();
    for (Map.Entry<String, LocalResource> rsrc : localResourceMap.entrySet()) {
        LocalResource resource = rsrc.getValue();
        LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
        allResources.putIfAbsent(req, new HashSet<>());
        allResources.get(req).add(rsrc.getKey());
        storeSharedCacheUploadPolicy(req, resource.getShouldBeUploadedToSharedCache());
        switch(resource.getVisibility()) {
            case PUBLIC:
                publicList.add(req);
                break;
            case PRIVATE:
                privateList.add(req);
                break;
            case APPLICATION:
                appList.add(req);
                break;
            default:
                break;
        }
    }
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new LinkedHashMap<>();
    if (!publicList.isEmpty()) {
        publicRsrcs.addAll(publicList);
        req.put(LocalResourceVisibility.PUBLIC, publicList);
    }
    if (!privateList.isEmpty()) {
        privateRsrcs.addAll(privateList);
        req.put(LocalResourceVisibility.PRIVATE, privateList);
    }
    if (!appList.isEmpty()) {
        appRsrcs.addAll(appList);
        req.put(LocalResourceVisibility.APPLICATION, appList);
    }
    if (!allResources.isEmpty()) {
        this.pendingResources.putAll(allResources);
    }
    return req;
}