Examples with Configuration org.apache.hadoop.conf.Configuration used on opensource projects

Example 21 with Configuration

use of org.apache.hadoop.conf.Configuration in project camel by apache.

the class HdfsConsumerTest method testReadFloat.

@Test
public void testReadFloat() throws Exception {
    if (!canTest()) {
        return;
    }
    final Path file = new Path(new File("target/test/test-camel-float").getAbsolutePath());
    Configuration conf = new Configuration();
    SequenceFile.Writer writer = createWriter(conf, file, NullWritable.class, FloatWritable.class);
    NullWritable keyWritable = NullWritable.get();
    FloatWritable valueWritable = new FloatWritable();
    float value = 3.1415926535f;
    valueWritable.set(value);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();
    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);
    context.addRoutes(new RouteBuilder() {

        public void configure() {
            from("hdfs2:localhost/" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0").to("mock:result");
        }
    });
    context.start();
    resultEndpoint.assertIsSatisfied();
}

Example 22 with Configuration

use of org.apache.hadoop.conf.Configuration in project camel by apache.

the class HdfsConsumerTest method testSimpleConsumerFileWithSizeEqualToNChunks.

@Test
public void testSimpleConsumerFileWithSizeEqualToNChunks() throws Exception {
    if (!canTest()) {
        return;
    }
    final Path file = new Path(new File("target/test/test-camel-normal-file").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(file.toUri(), conf);
    FSDataOutputStream out = fs.create(file);
    // size = 5 times chunk size = 210 bytes
    for (int i = 0; i < 42; ++i) {
        out.write(new byte[] { 0x61, 0x62, 0x63, 0x64, 0x65 });
        out.flush();
    }
    out.close();
    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(5);
    context.addRoutes(new RouteBuilder() {

        public void configure() {
            from("hdfs2:localhost/" + file.toUri() + "?fileSystemType=LOCAL&chunkSize=42&initialDelay=0").to("mock:result");
        }
    });
    context.start();
    resultEndpoint.assertIsSatisfied();
    assertThat(resultEndpoint.getReceivedExchanges().get(0).getIn().getBody(ByteArrayOutputStream.class).toByteArray().length, equalTo(42));
}

Example 23 with Configuration

use of org.apache.hadoop.conf.Configuration in project camel by apache.

the class HdfsConsumerTest method testReadBoolean.

@Test
public void testReadBoolean() throws Exception {
    if (!canTest()) {
        return;
    }
    final Path file = new Path(new File("target/test/test-camel-boolean").getAbsolutePath());
    Configuration conf = new Configuration();
    SequenceFile.Writer writer = createWriter(conf, file, NullWritable.class, BooleanWritable.class);
    NullWritable keyWritable = NullWritable.get();
    BooleanWritable valueWritable = new BooleanWritable();
    valueWritable.set(true);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();
    context.addRoutes(new RouteBuilder() {

        public void configure() {
            from("hdfs2:localhost/" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0").to("mock:result");
        }
    });
    context.start();
    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);
    resultEndpoint.assertIsSatisfied();
}

Example 24 with Configuration

use of org.apache.hadoop.conf.Configuration in project camel by apache.

the class HdfsProducerSplitTest method doTest.

private void doTest(int routeNr) throws Exception {
    if (!canTest()) {
        return;
    }
    for (int i = 0; i < 10; ++i) {
        template.sendBody("direct:start" + routeNr, "CIAO" + i);
    }
    stopCamelContext();
    FileSystem fs = FileSystem.get(new Configuration());
    FileStatus[] status = fs.listStatus(new Path("file:///" + BASE_FILE.toUri() + routeNr));
    assertEquals(10, status.length);
    for (FileStatus fileStatus : status) {
        BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(fileStatus.getPath())));
        assertTrue(br.readLine().startsWith("CIAO"));
        assertNull(br.readLine());
    }
}

Example 25 with Configuration

use of org.apache.hadoop.conf.Configuration in project camel by apache.

the class HdfsProducerSplitTest method tearDown.

@Override
public void tearDown() throws Exception {
    if (!canTest()) {
        return;
    }
    super.tearDown();
    Thread.sleep(100);
    Configuration conf = new Configuration();
    Path dir = new Path("target/test");
    FileSystem fs = FileSystem.get(dir.toUri(), conf);
    fs.delete(dir, true);
}