Examples with FileContext org.apache.hadoop.fs.FileContext used on opensource projects

Example 41 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class TestFSDownload method downloadWithFileType.

private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, URISyntaxException, InterruptedException {
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    Random rand = new Random();
    long sharedSeed = rand.nextLong();
    rand.setSeed(sharedSeed);
    System.out.println("SEED: " + sharedSeed);
    Map<LocalResource, Future<Path>> pending = new HashMap<LocalResource, Future<Path>>();
    ExecutorService exec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    int size = rand.nextInt(512) + 512;
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    Path p = new Path(basedir, "" + 1);
    String strFileName = "";
    LocalResource rsrc = null;
    switch(fileType) {
        case TAR:
            rsrc = createTarFile(files, p, size, rand, vis);
            break;
        case JAR:
            rsrc = createJarFile(files, p, size, rand, vis);
            rsrc.setType(LocalResourceType.PATTERN);
            break;
        case ZIP:
            rsrc = createZipFile(files, p, size, rand, vis);
            strFileName = p.getName() + ".ZIP";
            break;
        case TGZ:
            rsrc = createTgzFile(files, p, size, rand, vis);
            break;
    }
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    pending.put(rsrc, exec.submit(fsd));
    exec.shutdown();
    while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    try {
        // see if there was an Exception during download
        pending.get(rsrc).get();
        FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus(basedir);
        for (FileStatus filestatus : filesstatus) {
            if (filestatus.isDirectory()) {
                FileStatus[] childFiles = files.getDefaultFileSystem().listStatus(filestatus.getPath());
                for (FileStatus childfile : childFiles) {
                    if (strFileName.endsWith(".ZIP") && childfile.getPath().getName().equals(strFileName) && !childfile.isDirectory()) {
                        Assert.fail("Failure...After unzip, there should have been a" + " directory formed with zip file name but found a file. " + childfile.getPath());
                    }
                    if (childfile.getPath().getName().startsWith("tmp")) {
                        Assert.fail("Tmp File should not have been there " + childfile.getPath());
                    }
                }
            }
        }
    } catch (Exception e) {
        throw new IOException("Failed exec", e);
    }
}

Example 42 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class TestFSDownload method testDownloadPublicWithStatCache.

@Test(timeout = 60000)
public void testDownloadPublicWithStatCache() throws IOException, URISyntaxException, InterruptedException, ExecutionException {
    final Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    // if test directory doesn't have ancestor permission, skip this test
    FileSystem f = basedir.getFileSystem(conf);
    assumeTrue(FSDownload.ancestorsHaveExecutePermissions(f, basedir, null));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    int size = 512;
    final ConcurrentMap<Path, AtomicInteger> counts = new ConcurrentHashMap<Path, AtomicInteger>();
    final CacheLoader<Path, Future<FileStatus>> loader = FSDownload.createStatusCacheLoader(conf);
    final LoadingCache<Path, Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(new CacheLoader<Path, Future<FileStatus>>() {

        public Future<FileStatus> load(Path path) throws Exception {
            // increment the count
            AtomicInteger count = counts.get(path);
            if (count == null) {
                count = new AtomicInteger(0);
                AtomicInteger existing = counts.putIfAbsent(path, count);
                if (existing != null) {
                    count = existing;
                }
            }
            count.incrementAndGet();
            // use the default loader
            return loader.load(path);
        }
    });
    // test FSDownload.isPublic() concurrently
    final int fileCount = 3;
    List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
    for (int i = 0; i < fileCount; i++) {
        Random rand = new Random();
        long sharedSeed = rand.nextLong();
        rand.setSeed(sharedSeed);
        System.out.println("SEED: " + sharedSeed);
        final Path path = new Path(basedir, "test-file-" + i);
        createFile(files, path, size, rand);
        final FileSystem fs = path.getFileSystem(conf);
        final FileStatus sStat = fs.getFileStatus(path);
        tasks.add(new Callable<Boolean>() {

            public Boolean call() throws IOException {
                return FSDownload.isPublic(fs, path, sStat, statCache);
            }
        });
    }
    ExecutorService exec = HadoopExecutors.newFixedThreadPool(fileCount);
    try {
        List<Future<Boolean>> futures = exec.invokeAll(tasks);
        // files should be public
        for (Future<Boolean> future : futures) {
            assertTrue(future.get());
        }
        // for each path exactly one file status call should be made
        for (AtomicInteger count : counts.values()) {
            assertSame(count.get(), 1);
        }
    } finally {
        exec.shutdown();
    }
}

Example 43 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class DefaultContainerExecutor method startLocalizer.

@Override
public void startLocalizer(LocalizerStartContext ctx) throws IOException, InterruptedException {
    Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
    InetSocketAddress nmAddr = ctx.getNmAddr();
    String user = ctx.getUser();
    String appId = ctx.getAppId();
    String locId = ctx.getLocId();
    LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
    List<String> localDirs = dirsHandler.getLocalDirs();
    List<String> logDirs = dirsHandler.getLogDirs();
    createUserLocalDirs(localDirs, user);
    createUserCacheDirs(localDirs, user);
    createAppDirs(localDirs, user, appId);
    createAppLogDirs(appId, logDirs, user);
    // randomly choose the local directory
    Path appStorageDir = getWorkingDir(localDirs, user, appId);
    String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
    Path tokenDst = new Path(appStorageDir, tokenFn);
    copyFile(nmPrivateContainerTokensPath, tokenDst, user);
    LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
    FileContext localizerFc = FileContext.getFileContext(lfs.getDefaultFileSystem(), getConf());
    localizerFc.setUMask(lfs.getUMask());
    localizerFc.setWorkingDirectory(appStorageDir);
    LOG.info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.getWorkingDirectory());
    ContainerLocalizer localizer = createContainerLocalizer(user, appId, locId, localDirs, localizerFc);
    // TODO: DO it over RPC for maintaining similarity?
    localizer.runLocalization(nmAddr);
}

Example 44 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class ContainerLaunch method call.

@Override
// dispatcher not typed
@SuppressWarnings("unchecked")
public Integer call() {
    if (!validateContainerState()) {
        return 0;
    }
    final ContainerLaunchContext launchContext = container.getLaunchContext();
    ContainerId containerID = container.getContainerId();
    String containerIdStr = containerID.toString();
    final List<String> command = launchContext.getCommands();
    int ret = -1;
    Path containerLogDir;
    try {
        Map<Path, List<String>> localResources = getLocalizedResources();
        final String user = container.getUser();
        // /////////////////////////// Variable expansion
        // Before the container script gets written out.
        List<String> newCmds = new ArrayList<String>(command.size());
        String appIdStr = app.getAppId().toString();
        String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(appIdStr, containerIdStr);
        containerLogDir = dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
        recordContainerLogDir(containerID, containerLogDir.toString());
        for (String str : command) {
            // TODO: Should we instead work via symlinks without this grammar?
            newCmds.add(expandEnvironment(str, containerLogDir));
        }
        launchContext.setCommands(newCmds);
        Map<String, String> environment = launchContext.getEnvironment();
        // Make a copy of env to iterate & do variable expansion
        for (Entry<String, String> entry : environment.entrySet()) {
            String value = entry.getValue();
            value = expandEnvironment(value, containerLogDir);
            entry.setValue(value);
        }
        // /////////////////////////// End of variable expansion
        FileContext lfs = FileContext.getLocalFSFileContext();
        Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + CONTAINER_SCRIPT);
        Path nmPrivateTokensPath = dirsHandler.getLocalPathForWrite(getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, containerIdStr));
        Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(getContainerPrivateDir(appIdStr, containerIdStr));
        DataOutputStream containerScriptOutStream = null;
        DataOutputStream tokensOutStream = null;
        // Select the working directory for the container
        Path containerWorkDir = dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr + Path.SEPARATOR + containerIdStr, LocalDirAllocator.SIZE_UNKNOWN, false);
        recordContainerWorkDir(containerID, containerWorkDir.toString());
        String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
        // pid file should be in nm private dir so that it is not 
        // accessible by users
        pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
        List<String> localDirs = dirsHandler.getLocalDirs();
        List<String> logDirs = dirsHandler.getLogDirs();
        List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
        List<String> userLocalDirs = getUserLocalDirs(localDirs);
        List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
        List<String> containerLogDirs = getContainerLogDirs(logDirs);
        if (!dirsHandler.areDisksHealthy()) {
            ret = ContainerExitStatus.DISKS_FAILED;
            throw new IOException("Most of the disks failed. " + dirsHandler.getDisksHealthReport(false));
        }
        try {
            // /////////// Write out the container-script in the nmPrivate space.
            List<Path> appDirs = new ArrayList<Path>(localDirs.size());
            for (String localDir : localDirs) {
                Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
                Path userdir = new Path(usersdir, user);
                Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
                appDirs.add(new Path(appsdir, appIdStr));
            }
            containerScriptOutStream = lfs.create(nmPrivateContainerScriptPath, EnumSet.of(CREATE, OVERWRITE));
            // Set the token location too.
            environment.put(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME, new Path(containerWorkDir, FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
            // Sanitize the container's environment
            sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs, containerLogDirs, localResources, nmPrivateClasspathJarDir);
            exec.prepareContainer(new ContainerPrepareContext.Builder().setContainer(container).setLocalizedResources(localResources).setUser(user).setContainerLocalDirs(containerLocalDirs).setCommands(launchContext.getCommands()).build());
            // Write out the environment
            exec.writeLaunchEnv(containerScriptOutStream, environment, localResources, launchContext.getCommands(), new Path(containerLogDirs.get(0)), user);
            // /////////// End of writing out container-script
            // /////////// Write out the container-tokens in the nmPrivate space.
            tokensOutStream = lfs.create(nmPrivateTokensPath, EnumSet.of(CREATE, OVERWRITE));
            Credentials creds = container.getCredentials();
            creds.writeTokenStorageToStream(tokensOutStream);
        // /////////// End of writing out container-tokens
        } finally {
            IOUtils.cleanup(LOG, containerScriptOutStream, tokensOutStream);
        }
        ret = launchContainer(new ContainerStartContext.Builder().setContainer(container).setLocalizedResources(localResources).setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath).setNmPrivateTokensPath(nmPrivateTokensPath).setUser(user).setAppId(appIdStr).setContainerWorkDir(containerWorkDir).setLocalDirs(localDirs).setLogDirs(logDirs).setFilecacheDirs(filecacheDirs).setUserLocalDirs(userLocalDirs).setContainerLocalDirs(containerLocalDirs).setContainerLogDirs(containerLogDirs).build());
    } catch (Throwable e) {
        LOG.warn("Failed to launch container.", e);
        dispatcher.getEventHandler().handle(new ContainerExitEvent(containerID, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret, e.getMessage()));
        return ret;
    } finally {
        setContainerCompletedStatus(ret);
    }
    handleContainerExitCode(ret, containerLogDir);
    return ret;
}

Example 45 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class ContainerLaunch method cleanupContainer.

/**
   * Cleanup the container.
   * Cancels the launch if launch has not started yet or signals
   * the executor to not execute the process if not already done so.
   * Also, sends a SIGTERM followed by a SIGKILL to the process if
   * the process id is available.
   * @throws IOException
   */
// dispatcher not typed
@SuppressWarnings("unchecked")
public void cleanupContainer() throws IOException {
    ContainerId containerId = container.getContainerId();
    String containerIdStr = containerId.toString();
    LOG.info("Cleaning up container " + containerIdStr);
    try {
        context.getNMStateStore().storeContainerKilled(containerId);
    } catch (IOException e) {
        LOG.error("Unable to mark container " + containerId + " killed in store", e);
    }
    // launch flag will be set to true if process already launched
    boolean alreadyLaunched = !containerAlreadyLaunched.compareAndSet(false, true);
    if (!alreadyLaunched) {
        LOG.info("Container " + containerIdStr + " not launched." + " No cleanup needed to be done");
        return;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Marking container " + containerIdStr + " as inactive");
    }
    // this should ensure that if the container process has not launched 
    // by this time, it will never be launched
    exec.deactivateContainer(containerId);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Getting pid for container " + containerIdStr + " to kill" + " from pid file " + (pidFilePath != null ? pidFilePath.toString() : "null"));
    }
    // however the container process may have already started
    try {
        // get process id from pid file if available
        // else if shell is still active, get it from the shell
        String processId = null;
        if (pidFilePath != null) {
            processId = getContainerPid(pidFilePath);
        }
        // kill process
        if (processId != null) {
            String user = container.getUser();
            if (LOG.isDebugEnabled()) {
                LOG.debug("Sending signal to pid " + processId + " as user " + user + " for container " + containerIdStr);
            }
            final Signal signal = sleepDelayBeforeSigKill > 0 ? Signal.TERM : Signal.KILL;
            boolean result = exec.signalContainer(new ContainerSignalContext.Builder().setContainer(container).setUser(user).setPid(processId).setSignal(signal).build());
            if (LOG.isDebugEnabled()) {
                LOG.debug("Sent signal " + signal + " to pid " + processId + " as user " + user + " for container " + containerIdStr + ", result=" + (result ? "success" : "failed"));
            }
            if (sleepDelayBeforeSigKill > 0) {
                new DelayedProcessKiller(container, user, processId, sleepDelayBeforeSigKill, Signal.KILL, exec).start();
            }
        }
    } catch (Exception e) {
        String message = "Exception when trying to cleanup container " + containerIdStr + ": " + StringUtils.stringifyException(e);
        LOG.warn(message);
        dispatcher.getEventHandler().handle(new ContainerDiagnosticsUpdateEvent(containerId, message));
    } finally {
        // cleanup pid file if present
        if (pidFilePath != null) {
            FileContext lfs = FileContext.getLocalFSFileContext();
            lfs.delete(pidFilePath, false);
            lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
        }
    }
}