Examples with FileContext org.apache.hadoop.fs.FileContext used on opensource projects

Example 76 with FileContext

use of org.apache.hadoop.fs.FileContext in project tez by apache.

the class MiniTezCluster method serviceInit.

@Override
public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
    // Use libs from cluster since no build is available
    conf.setBoolean(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS, true);
    // blacklisting disabled to prevent scheduling issues
    conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
    }
    if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) {
        // nothing defined. set quick delete value
        conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
    }
    maxTimeToWaitForAppsOnShutdown = conf.getLong(TezConfiguration.TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS, TezConfiguration.TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS_DEFAULT);
    File appJarLocalFile = new File(MiniTezCluster.APPJAR);
    if (!appJarLocalFile.exists()) {
        String message = "TezAppJar " + MiniTezCluster.APPJAR + " not found. Exiting.";
        LOG.info(message);
        throw new TezUncheckedException(message);
    } else {
        LOG.info("Using Tez AppJar: " + appJarLocalFile.getAbsolutePath());
    }
    FileSystem fs = FileSystem.get(conf);
    Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir"));
    Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar");
    // Copy AppJar and make it public.
    Path appMasterJar = new Path(MiniTezCluster.APPJAR);
    fs.copyFromLocalFile(appMasterJar, appRemoteJar);
    fs.setPermission(appRemoteJar, new FsPermission("777"));
    conf.set(TezConfiguration.TEZ_LIB_URIS, appRemoteJar.toUri().toString());
    LOG.info("Set TEZ-LIB-URI to: " + conf.get(TezConfiguration.TEZ_LIB_URIS));
    // VMEM monitoring disabled, PMEM monitoring enabled.
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
    try {
        Path stagingPath = FileContext.getFileContext(conf).makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        /*
       * Re-configure the staging path on Windows if the file system is localFs.
       * We need to use a absolute path that contains the drive letter. The unit
       * test could run on a different drive than the AM. We can run into the
       * issue that job files are localized to the drive where the test runs on,
       * while the AM starts on a different drive and fails to find the job
       * metafiles. Using absolute path can avoid this ambiguity.
       */
        if (Path.WINDOWS) {
            if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
                conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath());
            }
        }
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        fc.mkdir(stagingPath, null, true);
        // mkdir done directory as well
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new TezUncheckedException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test");
    // configure the shuffle service in NM
    if (conf.get(YarnConfiguration.NM_AUX_SERVICES) == null) {
        conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
        conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class);
        // Non-standard shuffle port
        conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    }
    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class);
    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    super.serviceInit(conf);
}

Example 77 with FileContext

use of org.apache.hadoop.fs.FileContext in project cdap by caskdata.

the class CoprocessorBuildTool method main.

public static void main(final String[] args) throws ParseException {
    Options options = new Options().addOption(new Option("h", "help", false, "Print this usage message.")).addOption(new Option("f", "force", false, "Overwrites any coprocessors that already exist."));
    CommandLineParser parser = new BasicParser();
    CommandLine commandLine = parser.parse(options, args);
    String[] commandArgs = commandLine.getArgs();
    // if help is an option, or if there isn't a single 'ensure' command, print usage and exit.
    if (commandLine.hasOption("h") || commandArgs.length != 1 || !"check".equalsIgnoreCase(commandArgs[0])) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp(CoprocessorBuildTool.class.getName() + " check", "Checks that HBase coprocessors required by CDAP are loaded onto HDFS. " + "If not, the coprocessors are built and placed on HDFS.", options, "");
        System.exit(0);
    }
    boolean overwrite = commandLine.hasOption("f");
    CConfiguration cConf = CConfiguration.create();
    Configuration hConf = HBaseConfiguration.create();
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), // for LocationFactory
    new PrivateModule() {

        @Override
        protected void configure() {
            bind(FileContext.class).toProvider(FileContextProvider.class).in(Scopes.SINGLETON);
            expose(LocationFactory.class);
        }

        @Provides
        @Singleton
        private LocationFactory providesLocationFactory(Configuration hConf, CConfiguration cConf, FileContext fc) {
            final String namespace = cConf.get(Constants.CFG_HDFS_NAMESPACE);
            if (UserGroupInformation.isSecurityEnabled()) {
                return new FileContextLocationFactory(hConf, namespace);
            }
            return new InsecureFileContextLocationFactory(hConf, namespace, fc);
        }
    });
    try {
        SecurityUtil.loginForMasterService(cConf);
    } catch (Exception e) {
        LOG.error("Failed to login as CDAP user", e);
        System.exit(1);
    }
    LocationFactory locationFactory = injector.getInstance(LocationFactory.class);
    HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
    CoprocessorManager coprocessorManager = new CoprocessorManager(cConf, locationFactory, tableUtil);
    try {
        Location location = coprocessorManager.ensureCoprocessorExists(overwrite);
        LOG.info("coprocessor exists at {}.", location);
    } catch (IOException e) {
        LOG.error("Unable to build and upload coprocessor jars.", e);
        System.exit(1);
    }
}

Example 78 with FileContext

use of org.apache.hadoop.fs.FileContext in project apex-core by apache.

the class CascadeStorageAgentTest method testSingleIndirection.

@Test
public void testSingleIndirection() throws IOException {
    String oldAppPath = testMeta.applicationPath;
    FSStorageAgent storageAgent = new FSStorageAgent(oldAppPath, null);
    storageAgent.save("1", 1, 1);
    storageAgent.save("2", 1, 2);
    storageAgent.save("3", 2, 1);
    String newAppPath = oldAppPath + ".new";
    CascadeStorageAgent cascade = new CascadeStorageAgent(storageAgent, new FSStorageAgent(newAppPath, null));
    long[] operatorIds = cascade.getWindowIds(1);
    Assert.assertArrayEquals("Returned window ids ", operatorIds, new long[] { 1L, 2L });
    operatorIds = cascade.getWindowIds(2);
    Assert.assertArrayEquals("Returned window ids ", operatorIds, new long[] { 1L });
    /* save should happen to new location */
    cascade.save("4", 1, 4);
    FileContext fileContext = FileContext.getFileContext();
    Assert.assertFalse("operator 1 window 4 file does not exists in old directory", fileContext.util().exists(new Path(oldAppPath + "/" + 1 + "/" + 4)));
    Assert.assertTrue("operator 1 window 4 file exists in new directory", fileContext.util().exists(new Path(newAppPath + "/" + 1 + "/" + 4)));
    // check for delete,
    // delete for old checkpoint should be ignored
    cascade.save("5", 1, 5);
    cascade.delete(1, 2L);
    Assert.assertTrue("operator 1 window 2 file exists in old directory", fileContext.util().exists(new Path(oldAppPath + "/" + 1 + "/" + 2)));
    cascade.delete(1, 4L);
    Assert.assertFalse("operator 1 window 4 file does not exists in old directory", fileContext.util().exists(new Path(newAppPath + "/" + 1 + "/" + 4)));
    /* chaining of storage agent */
    String latestAppPath = oldAppPath + ".latest";
    cascade = new CascadeStorageAgent(storageAgent, new FSStorageAgent(newAppPath, null));
    CascadeStorageAgent latest = new CascadeStorageAgent(cascade, new FSStorageAgent(latestAppPath, null));
    operatorIds = latest.getWindowIds(1);
    Assert.assertArrayEquals("Window ids ", operatorIds, new long[] { 1, 2, 5 });
    latest.save("6", 1, 6);
    Assert.assertFalse("operator 1 window 6 file does not exists in old directory", fileContext.util().exists(new Path(oldAppPath + "/" + 1 + "/" + 6)));
    Assert.assertFalse("operator 1 window 6 file does not exists in old directory", fileContext.util().exists(new Path(newAppPath + "/" + 1 + "/" + 6)));
    Assert.assertTrue("operator 1 window 6 file exists in new directory", fileContext.util().exists(new Path(latestAppPath + "/" + 1 + "/" + 6)));
}

Example 79 with FileContext

use of org.apache.hadoop.fs.FileContext in project apex-core by apache.

the class FSRecoveryHandler method restore.

@Override
public Object restore() throws IOException {
    FileContext fc = FileContext.getFileContext(fs.getUri());
    // recover from wherever it was left
    if (fc.util().exists(snapshotBackupPath)) {
        LOG.warn("Incomplete checkpoint, reverting to {}", snapshotBackupPath);
        fc.rename(snapshotBackupPath, snapshotPath, Rename.OVERWRITE);
        // combine logs (w/o append, create new file)
        Path tmpLogPath = new Path(basedir, "log.combined");
        try (FSDataOutputStream fsOut = fc.create(tmpLogPath, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE))) {
            try (FSDataInputStream fsIn = fc.open(logBackupPath)) {
                IOUtils.copy(fsIn, fsOut);
            }
            try (FSDataInputStream fsIn = fc.open(logPath)) {
                IOUtils.copy(fsIn, fsOut);
            }
        }
        fc.rename(tmpLogPath, logPath, Rename.OVERWRITE);
        fc.delete(logBackupPath, false);
    } else {
        // failure between log rotation and writing checkpoint
        if (fc.util().exists(logBackupPath)) {
            LOG.warn("Found {}, did checkpointing fail?", logBackupPath);
            fc.rename(logBackupPath, logPath, Rename.OVERWRITE);
        }
    }
    if (!fc.util().exists(snapshotPath)) {
        LOG.debug("No existing checkpoint.");
        return null;
    }
    LOG.debug("Reading checkpoint {}", snapshotPath);
    InputStream is = fc.open(snapshotPath);
    // indeterministic class loading behavior
    // http://stackoverflow.com/questions/9110677/readresolve-not-working-an-instance-of-guavas-serializedform-appears
    final ClassLoader loader = Thread.currentThread().getContextClassLoader();
    try (ObjectInputStream ois = new ObjectInputStream(is) {

        @Override
        protected Class<?> resolveClass(ObjectStreamClass objectStreamClass) throws IOException, ClassNotFoundException {
            return Class.forName(objectStreamClass.getName(), true, loader);
        }
    }) {
        return ois.readObject();
    } catch (ClassNotFoundException cnfe) {
        throw new IOException("Failed to read checkpointed state", cnfe);
    }
}

Example 80 with FileContext

use of org.apache.hadoop.fs.FileContext in project jstorm by alibaba.

the class HdfsBlobStoreFile method commit.

@Override
public void commit() throws IOException {
    checkIsNotTmp();
    // FileContext supports atomic rename, whereas FileSystem doesn't
    FileContext fc = FileContext.getFileContext(_hadoopConf);
    Path dest = new Path(_path.getParent(), BLOBSTORE_DATA_FILE);
    if (_mustBeNew) {
        fc.rename(_path, dest);
    } else {
        fc.rename(_path, dest, Options.Rename.OVERWRITE);
    }
// Note, we could add support for setting the replication factor
}