Examples with LocalFileSystem org.apache.hadoop.fs.LocalFileSystem used on opensource projects

Example 51 with LocalFileSystem

use of org.apache.hadoop.fs.LocalFileSystem in project druid by druid-io.

the class HdfsDataSegmentPullerTest method setupStatic.

@BeforeClass
public static void setupStatic() throws IOException {
    hdfsTmpDir = File.createTempFile("hdfsHandlerTest", "dir");
    if (!hdfsTmpDir.delete()) {
        throw new IOE("Unable to delete hdfsTmpDir [%s]", hdfsTmpDir.getAbsolutePath());
    }
    conf = new Configuration(true);
    fileSystem = new LocalFileSystem();
    fileSystem.initialize(hdfsTmpDir.toURI(), conf);
    fileSystem.setWorkingDirectory(new Path(hdfsTmpDir.toURI()));
    final File tmpFile = File.createTempFile("hdfsHandlerTest", ".data");
    tmpFile.delete();
    try {
        Files.copy(new ByteArrayInputStream(pathByteContents), tmpFile.toPath());
        try (OutputStream stream = fileSystem.create(filePath)) {
            Files.copy(tmpFile.toPath(), stream);
        }
    } finally {
        tmpFile.delete();
    }
}

Example 52 with LocalFileSystem

use of org.apache.hadoop.fs.LocalFileSystem in project hive by apache.

the class TestSystemVariables method test_RelativeJavaIoTmpDir_CoercedTo_AbsolutePath.

@Test
public void test_RelativeJavaIoTmpDir_CoercedTo_AbsolutePath() {
    FileSystem localFileSystem = new LocalFileSystem();
    String systemJavaIoTmpDir = makeVarName(SYSTEM, "java.io.tmpdir");
    System.setProperty("java.io.tmpdir", "./relativePath");
    Path relativePath = new Path(localFileSystem.getWorkingDirectory(), "./relativePath");
    assertEquals(relativePath.toString(), SystemVariables.substitute(systemJavaIoTmpDir));
    System.setProperty("java.io.tmpdir", "this/is/a/relative/path");
    Path thisIsARelativePath = new Path(localFileSystem.getWorkingDirectory(), "this/is/a/relative/path");
    assertEquals(thisIsARelativePath.toString(), SystemVariables.substitute(systemJavaIoTmpDir));
}

Example 53 with LocalFileSystem

use of org.apache.hadoop.fs.LocalFileSystem in project hive by apache.

the class HCatMapReduceTest method setUpOneTime.

@BeforeClass
public static void setUpOneTime() throws Exception {
    fs = new LocalFileSystem();
    fs.initialize(fs.getWorkingDirectory().toUri(), new Configuration());
    HiveConf hiveConf = new HiveConf();
    hiveConf.setInt(HCatConstants.HCAT_HIVE_CLIENT_EXPIRY_TIME, 0);
    // Hack to initialize cache with 0 expiry time causing it to return a new hive client every time
    // Otherwise the cache doesn't play well with the second test method with the client gets closed() in the
    // tearDown() of the previous test
    HCatUtil.getHiveMetastoreClient(hiveConf);
    MapCreate.writeCount = 0;
    MapRead.readCount = 0;
}

Example 54 with LocalFileSystem

use of org.apache.hadoop.fs.LocalFileSystem in project hive by apache.

the class TestFileUtils method testRelativePathToAbsolutePath.

@Test
public void testRelativePathToAbsolutePath() throws IOException {
    LocalFileSystem localFileSystem = new LocalFileSystem();
    Path actualPath = FileUtils.makeAbsolute(localFileSystem, new Path("relative/path"));
    Path expectedPath = new Path(localFileSystem.getWorkingDirectory(), "relative/path");
    assertEquals(expectedPath.toString(), actualPath.toString());
    Path absolutePath = new Path("/absolute/path");
    Path unchangedPath = FileUtils.makeAbsolute(localFileSystem, new Path("/absolute/path"));
    assertEquals(unchangedPath.toString(), absolutePath.toString());
}

Example 55 with LocalFileSystem

use of org.apache.hadoop.fs.LocalFileSystem in project hive by apache.

the class TestHCatLoaderEncryption method testReadDataFromEncryptedHiveTableByHCatMR.

@Test
public void testReadDataFromEncryptedHiveTableByHCatMR() throws Exception {
    assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS));
    readRecords.clear();
    Configuration conf = new Configuration();
    Job job = new Job(conf, "hcat mapreduce read encryption test");
    job.setJarByClass(this.getClass());
    job.setMapperClass(TestHCatLoaderEncryption.MapRead.class);
    // input/output settings
    job.setInputFormatClass(HCatInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    HCatInputFormat.setInput(job, Warehouse.DEFAULT_DATABASE_NAME, ENCRYPTED_TABLE, null);
    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setNumReduceTasks(0);
    FileSystem fs = new LocalFileSystem();
    String pathLoc = TEST_DATA_DIR + "/testHCatMREncryptionOutput";
    Path path = new Path(pathLoc);
    if (fs.exists(path)) {
        fs.delete(path, true);
    }
    TextOutputFormat.setOutputPath(job, new Path(pathLoc));
    job.waitForCompletion(true);
    int numTuplesRead = 0;
    for (HCatRecord hCatRecord : readRecords) {
        assertEquals(2, hCatRecord.size());
        assertNotNull(hCatRecord.get(0));
        assertNotNull(hCatRecord.get(1));
        assertTrue(hCatRecord.get(0).getClass() == Integer.class);
        assertTrue(hCatRecord.get(1).getClass() == String.class);
        assertEquals(hCatRecord.get(0), basicInputData.get(numTuplesRead).first);
        assertEquals(hCatRecord.get(1), basicInputData.get(numTuplesRead).second);
        numTuplesRead++;
    }
    assertEquals("failed HCat MR read with storage format: " + this.storageFormat, basicInputData.size(), numTuplesRead);
}