Examples with FileContext org.apache.hadoop.fs.FileContext used on opensource projects

Example 6 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class TestLocalDirsHandlerService method testGetFullDirs.

@Test
public void testGetFullDirs() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    FileContext localFs = FileContext.getLocalFSFileContext(conf);
    String localDir1 = new File(testDir, "localDir1").getPath();
    String localDir2 = new File(testDir, "localDir2").getPath();
    String logDir1 = new File(testDir, "logDir1").getPath();
    String logDir2 = new File(testDir, "logDir2").getPath();
    Path localDir1Path = new Path(localDir1);
    Path logDir1Path = new Path(logDir1);
    FsPermission dirPermissions = new FsPermission((short) 0410);
    localFs.mkdir(localDir1Path, dirPermissions, true);
    localFs.mkdir(logDir1Path, dirPermissions, true);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1 + "," + logDir2);
    conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 0.0f);
    NodeManagerMetrics nm = NodeManagerMetrics.create();
    LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(nm);
    dirSvc.init(conf);
    Assert.assertEquals(0, dirSvc.getLocalDirs().size());
    Assert.assertEquals(0, dirSvc.getLogDirs().size());
    Assert.assertEquals(1, dirSvc.getDiskFullLocalDirs().size());
    Assert.assertEquals(1, dirSvc.getDiskFullLogDirs().size());
    // check the metrics
    Assert.assertEquals(2, nm.getBadLocalDirs());
    Assert.assertEquals(2, nm.getBadLogDirs());
    Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
    Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());
    Assert.assertEquals("", dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
    Assert.assertEquals("", dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
    Assert.assertEquals(localDir1 + "," + localDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
    Assert.assertEquals(logDir1 + "," + logDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
    conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100.0f);
    nm = NodeManagerMetrics.create();
    dirSvc = new LocalDirsHandlerService(nm);
    dirSvc.init(conf);
    Assert.assertEquals(1, dirSvc.getLocalDirs().size());
    Assert.assertEquals(1, dirSvc.getLogDirs().size());
    Assert.assertEquals(0, dirSvc.getDiskFullLocalDirs().size());
    Assert.assertEquals(0, dirSvc.getDiskFullLogDirs().size());
    // check the metrics
    File dir = new File(localDir1);
    int utilizationPerc = (int) ((dir.getTotalSpace() - dir.getUsableSpace()) * 100 / dir.getTotalSpace());
    Assert.assertEquals(1, nm.getBadLocalDirs());
    Assert.assertEquals(1, nm.getBadLogDirs());
    Assert.assertEquals(utilizationPerc, nm.getGoodLocalDirsDiskUtilizationPerc());
    Assert.assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());
    Assert.assertEquals(localDir2, dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
    Assert.assertEquals(logDir2, dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
    Assert.assertEquals(localDir1 + "," + localDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
    Assert.assertEquals(logDir1 + "," + logDir2, dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
    FileUtils.deleteDirectory(new File(localDir1));
    FileUtils.deleteDirectory(new File(localDir2));
    FileUtils.deleteDirectory(new File(logDir1));
    FileUtils.deleteDirectory(new File(logDir2));
    dirSvc.close();
}

Example 7 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class MiniMRYarnCluster method serviceInit.

@Override
public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath());
    }
    // By default, VMEM monitoring disabled, PMEM monitoring enabled.
    if (!conf.getBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
        conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
        conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
    }
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
    try {
        Path stagingPath = FileContext.getFileContext(conf).makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        /*
       * Re-configure the staging path on Windows if the file system is localFs.
       * We need to use a absolute path that contains the drive letter. The unit
       * test could run on a different drive than the AM. We can run into the
       * issue that job files are localized to the drive where the test runs on,
       * while the AM starts on a different drive and fails to find the job
       * metafiles. Using absolute path can avoid this ambiguity.
       */
        if (Path.WINDOWS) {
            if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
                conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath());
            }
        }
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        //mkdir the staging directory so that right permissions are set while running as proxy user
        fc.mkdir(stagingPath, null, true);
        //mkdir done directory as well 
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new YarnRuntimeException("Could not create staging directory. ", e);
    }
    // The default is local because of
    conf.set(MRConfig.MASTER_ADDRESS, "test");
    // which shuffle doesn't happen
    //configure the shuffle service in NM
    String[] nmAuxServices = conf.getStrings(YarnConfiguration.NM_AUX_SERVICES);
    // if need to enable TIMELINE_AUX_SERVICE_NAME
    boolean enableTimelineAuxService = false;
    if (nmAuxServices != null) {
        for (String nmAuxService : nmAuxServices) {
            if (nmAuxService.equals(TIMELINE_AUX_SERVICE_NAME)) {
                enableTimelineAuxService = true;
                break;
            }
        }
    }
    if (enableTimelineAuxService) {
        conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, TIMELINE_AUX_SERVICE_NAME });
    } else {
        conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
    }
    conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class);
    // Non-standard shuffle port
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class);
    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    super.serviceInit(conf);
}

Example 8 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class TestNonAggregatingLogHandler method testLogDeletion.

@Test
public void testLogDeletion() throws IOException {
    File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
    String localLogDirsString = localLogDirs[0].getAbsolutePath() + "," + localLogDirs[1].getAbsolutePath();
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 0l);
    dirsHandler.init(conf);
    NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler, new NMNullStateStoreService());
    NonAggregatingLogHandler logHandler = spy(rawLogHandler);
    AbstractFileSystem spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
    FileContext lfs = FileContext.getFileContext(spylfs, conf);
    doReturn(lfs).when(logHandler).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(localLogDirs[0].getAbsolutePath()));
    doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));
    logHandler.init(conf);
    logHandler.start();
    logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
    logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    Path[] localAppLogDirs = new Path[2];
    localAppLogDirs[0] = new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
    localAppLogDirs[1] = new Path(localLogDirs[1].getAbsolutePath(), appId.toString());
    testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
    logHandler.close();
    for (int i = 0; i < localLogDirs.length; i++) {
        FileUtils.deleteDirectory(localLogDirs[i]);
    }
}

Example 9 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class JobSubmitter method addMRFrameworkToDistributedCache.

@SuppressWarnings("deprecation")
private static void addMRFrameworkToDistributedCache(Configuration conf) throws IOException {
    String framework = conf.get(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, "");
    if (!framework.isEmpty()) {
        URI uri;
        try {
            uri = new URI(framework);
        } catch (URISyntaxException e) {
            throw new IllegalArgumentException("Unable to parse '" + framework + "' as a URI, check the setting for " + MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, e);
        }
        String linkedName = uri.getFragment();
        // resolve any symlinks in the URI path so using a "current" symlink
        // to point to a specific version shows the specific version
        // in the distributed cache configuration
        FileSystem fs = FileSystem.get(conf);
        Path frameworkPath = fs.makeQualified(new Path(uri.getScheme(), uri.getAuthority(), uri.getPath()));
        FileContext fc = FileContext.getFileContext(frameworkPath.toUri(), conf);
        frameworkPath = fc.resolvePath(frameworkPath);
        uri = frameworkPath.toUri();
        try {
            uri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), null, linkedName);
        } catch (URISyntaxException e) {
            throw new IllegalArgumentException(e);
        }
        DistributedCache.addCacheArchive(uri, conf);
    }
}

Example 10 with FileContext

use of org.apache.hadoop.fs.FileContext in project hadoop by apache.

the class TestFSDownload method testUniqueDestinationPath.

@Test(timeout = 10000)
public void testUniqueDestinationPath() throws Exception {
    Configuration conf = new Configuration();
    FileContext files = FileContext.getLocalFSFileContext(conf);
    final Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName()));
    files.mkdir(basedir, null, true);
    conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
    ExecutorService singleThreadedExec = HadoopExecutors.newSingleThreadExecutor();
    LocalDirAllocator dirs = new LocalDirAllocator(TestFSDownload.class.getName());
    Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
    destPath = new Path(destPath, Long.toString(uniqueNumberGenerator.incrementAndGet()));
    Path p = new Path(basedir, "dir" + 0 + ".jar");
    LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
    LocalResource rsrc = createJar(files, p, vis);
    FSDownload fsd = new FSDownload(files, UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
    Future<Path> rPath = singleThreadedExec.submit(fsd);
    singleThreadedExec.shutdown();
    while (!singleThreadedExec.awaitTermination(1000, TimeUnit.MILLISECONDS)) ;
    Assert.assertTrue(rPath.isDone());
    // Now FSDownload will not create a random directory to localize the
    // resource. Therefore the final localizedPath for the resource should be
    // destination directory (passed as an argument) + file name.
    Assert.assertEquals(destPath, rPath.get().getParent());
}