Examples with FileContext org.apache.hadoop.fs.FileContext used on opensource projects

Example 81 with FileContext

use of org.apache.hadoop.fs.FileContext in project metron by apache.

the class YarnComponent method start.

@Override
public void start() throws UnableToStartException {
    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.set("yarn.log.dir", "target");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    try {
        yarnCluster = new MiniYARNCluster(testName, 1, NUM_NMS, 1, 1, true);
        yarnCluster.init(conf);
        yarnCluster.start();
        waitForNMsToRegister();
        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
        // write the document to a buffer (not directly to the file, as that
        // can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        // write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
        FileContext fsContext = FileContext.getLocalFSFileContext();
        fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
        }
    } catch (Exception e) {
        throw new UnableToStartException("Exception setting up yarn cluster", e);
    }
}

Example 82 with FileContext

use of org.apache.hadoop.fs.FileContext in project metron by apache.

the class YarnComponent method stop.

@Override
public void stop() {
    if (yarnCluster != null) {
        try {
            yarnCluster.stop();
        } finally {
            yarnCluster = null;
        }
    }
    try {
        FileContext fsContext = FileContext.getLocalFSFileContext();
        fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
    } catch (Exception e) {
    }
}

Example 83 with FileContext

use of org.apache.hadoop.fs.FileContext in project apex-malhar by apache.

the class FileAccessFSImpl method rename.

@Override
public void rename(long bucketKey, String fromName, String toName) throws IOException {
    FileContext fc = FileContext.getFileContext(fs.getUri());
    Path bucketPath = getBucketPath(bucketKey);
    // file context requires absolute path
    if (!bucketPath.isAbsolute()) {
        bucketPath = new Path(fs.getWorkingDirectory(), bucketPath);
    }
    fc.rename(new Path(bucketPath, fromName), new Path(bucketPath, toName), Rename.OVERWRITE);
}

Example 84 with FileContext

use of org.apache.hadoop.fs.FileContext in project apex-malhar by apache.

the class FileSystemWALTest method testDeleteEverything.

@Test
public void testDeleteEverything() throws IOException {
    testMeta.fsWAL.setMaxLength(2 * 1024);
    testMeta.fsWAL.setup();
    FileSystemWAL.FileSystemWALWriter fsWALWriter = testMeta.fsWAL.getWriter();
    write1KRecords(fsWALWriter, 10);
    testMeta.fsWAL.beforeCheckpoint(0);
    testMeta.fsWAL.committed(0);
    FileSystemWAL.FileSystemWALReader fsWALReader = testMeta.fsWAL.getReader();
    assertNumTuplesRead(fsWALReader, 10);
    FileSystemWAL.FileSystemWALPointer writerPointer = fsWALWriter.getPointer();
    fsWALWriter.delete(writerPointer);
    FileContext fileContext = FileContextUtils.getFileContext(testMeta.fsWAL.getFilePath());
    for (int i = 0; i < 5; i++) {
        Assert.assertTrue("part exists " + i, !fileContext.util().exists(new Path(testMeta.fsWAL.getPartFilePath(i))));
    }
    testMeta.fsWAL.teardown();
}