Examples with FileSystem org.apache.hadoop.fs.FileSystem used on opensource projects

Example 16 with FileSystem

use of org.apache.hadoop.fs.FileSystem in project camel by apache.

the class HdfsProducerSplitTest method doTest.

private void doTest(int routeNr) throws Exception {
    if (!canTest()) {
        return;
    }
    for (int i = 0; i < 10; ++i) {
        template.sendBody("direct:start" + routeNr, "CIAO" + i);
    }
    stopCamelContext();
    FileSystem fs = FileSystem.get(new Configuration());
    FileStatus[] status = fs.listStatus(new Path("file:///" + BASE_FILE.toUri() + routeNr));
    assertEquals(10, status.length);
    for (FileStatus fileStatus : status) {
        BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(fileStatus.getPath())));
        assertTrue(br.readLine().startsWith("CIAO"));
        assertNull(br.readLine());
    }
}

Example 17 with FileSystem

use of org.apache.hadoop.fs.FileSystem in project camel by apache.

the class HdfsProducerSplitTest method tearDown.

@Override
public void tearDown() throws Exception {
    if (!canTest()) {
        return;
    }
    super.tearDown();
    Thread.sleep(100);
    Configuration conf = new Configuration();
    Path dir = new Path("target/test");
    FileSystem fs = FileSystem.get(dir.toUri(), conf);
    fs.delete(dir, true);
}

Example 18 with FileSystem

use of org.apache.hadoop.fs.FileSystem in project camel by apache.

the class HdfsProducerTest method testWriteTextWithKey.

@Test
public void testWriteTextWithKey() throws Exception {
    if (!canTest()) {
        return;
    }
    String txtKey = "THEKEY";
    String txtValue = "CIAO MONDO !";
    template.sendBodyAndHeader("direct:write_text2", txtValue, "KEY", txtKey);
    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + TEMP_DIR.toUri() + "/test-camel-text2");
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf);
    Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);
    reader.next(key, value);
    assertEquals(key.toString(), txtKey);
    assertEquals(value.toString(), txtValue);
    IOHelper.close(reader);
}

Example 19 with FileSystem

use of org.apache.hadoop.fs.FileSystem in project camel by apache.

the class HdfsProducerTest method testArrayWriteText.

@Test
public void testArrayWriteText() throws Exception {
    if (!canTest()) {
        return;
    }
    String txtValue = "CIAO MONDO !";
    template.sendBody("direct:write_text4", txtValue);
    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + TEMP_DIR.toUri() + "/test-camel-text4");
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    ArrayFile.Reader reader = new ArrayFile.Reader(fs1, "file:///" + TEMP_DIR.toUri() + "/test-camel-text4", conf);
    Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);
    reader.next(value);
    assertEquals(value.toString(), txtValue);
    IOHelper.close(reader);
}

Example 20 with FileSystem

use of org.apache.hadoop.fs.FileSystem in project camel by apache.

the class HdfsProducerTest method testProducer.

@Test
public void testProducer() throws Exception {
    if (!canTest()) {
        return;
    }
    template.sendBody("direct:start1", "PAPPO");
    Configuration conf = new Configuration();
    Path file1 = new Path("file:///" + TEMP_DIR.toUri() + "/test-camel1");
    FileSystem fs1 = FileSystem.get(file1.toUri(), conf);
    SequenceFile.Reader reader = new SequenceFile.Reader(fs1, file1, conf);
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
    Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
    reader.next(key, value);
    assertEquals("PAPPO", value.toString());
    IOHelper.close(reader);
}