Examples with NullWritable org.apache.hadoop.io.NullWritable used on opensource projects

Example 21 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class AbstractNodeTupleOutputFormatTests method testOutput.

/**
     * Tests output
     * 
     * @param f
     *            File to output to
     * @param num
     *            Number of tuples to output
     * @throws IOException
     * @throws InterruptedException
     */
protected final void testOutput(File f, int num) throws IOException, InterruptedException {
    // Prepare configuration
    Configuration config = this.prepareConfiguration();
    // Set up fake job
    OutputFormat<NullWritable, T> outputFormat = this.getOutputFormat();
    Job job = Job.getInstance(config);
    job.setOutputFormatClass(outputFormat.getClass());
    this.addOutputPath(f, job.getConfiguration(), job);
    JobContext context = new JobContextImpl(job.getConfiguration(), job.getJobID());
    Assert.assertNotNull(FileOutputFormat.getOutputPath(context));
    // Output the data
    TaskAttemptID id = new TaskAttemptID("outputTest", 1, TaskType.MAP, 1, 1);
    TaskAttemptContext taskContext = new TaskAttemptContextImpl(job.getConfiguration(), id);
    RecordWriter<NullWritable, T> writer = outputFormat.getRecordWriter(taskContext);
    Iterator<T> tuples = this.generateTuples(num);
    while (tuples.hasNext()) {
        writer.write(NullWritable.get(), tuples.next());
    }
    writer.close(taskContext);
    // Check output
    File outputFile = this.findOutputFile(this.folder.getRoot(), context);
    Assert.assertNotNull(outputFile);
    this.checkTuples(outputFile, num);
}

Example 22 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class TestHadoopRdfIORegistry method testLang.

private void testLang(Lang lang, boolean triples, boolean quads, boolean writesSupported) {
    Assert.assertEquals(triples, HadoopRdfIORegistry.hasTriplesReader(lang));
    Assert.assertEquals(quads, HadoopRdfIORegistry.hasQuadReader(lang));
    // Some formats may be asymmetric
    if (writesSupported) {
        Assert.assertEquals(triples, HadoopRdfIORegistry.hasTriplesWriter(lang));
        Assert.assertEquals(quads, HadoopRdfIORegistry.hasQuadWriter(lang));
    } else {
        Assert.assertFalse(HadoopRdfIORegistry.hasTriplesWriter(lang));
        Assert.assertFalse(HadoopRdfIORegistry.hasQuadWriter(lang));
    }
    if (triples) {
        // Check that triples are supported
        RecordReader<LongWritable, TripleWritable> tripleReader;
        try {
            tripleReader = HadoopRdfIORegistry.createTripleReader(lang);
            Assert.assertNotNull(tripleReader);
        } catch (IOException e) {
            Assert.fail("Registry indicates that " + lang.getName() + " can read triples but fails to produce a triple reader when asked: " + e.getMessage());
        }
        if (writesSupported) {
            RecordWriter<NullWritable, TripleWritable> tripleWriter;
            try {
                tripleWriter = HadoopRdfIORegistry.createTripleWriter(lang, new StringWriter(), new Configuration(false));
                Assert.assertNotNull(tripleWriter);
            } catch (IOException e) {
                Assert.fail("Registry indicates that " + lang.getName() + " can write triples but fails to produce a triple writer when asked: " + e.getMessage());
            }
        }
    } else {
        // Check that triples are not supported
        try {
            HadoopRdfIORegistry.createTripleReader(lang);
            Assert.fail("Registry indicates that " + lang.getName() + " cannot read triples but produced a triple reader when asked (error was expected)");
        } catch (IOException e) {
        // This is expected
        }
        try {
            HadoopRdfIORegistry.createTripleWriter(lang, new StringWriter(), new Configuration(false));
            Assert.fail("Registry indicates that " + lang.getName() + " cannot write triples but produced a triple write when asked (error was expected)");
        } catch (IOException e) {
        // This is expected
        }
    }
    if (quads) {
        // Check that quads are supported
        RecordReader<LongWritable, QuadWritable> quadReader;
        try {
            quadReader = HadoopRdfIORegistry.createQuadReader(lang);
            Assert.assertNotNull(quadReader);
        } catch (IOException e) {
            Assert.fail("Registry indicates that " + lang.getName() + " can read quads but fails to produce a quad reader when asked: " + e.getMessage());
        }
        if (writesSupported) {
            RecordWriter<NullWritable, QuadWritable> quadWriter;
            try {
                quadWriter = HadoopRdfIORegistry.createQuadWriter(lang, new StringWriter(), new Configuration(false));
                Assert.assertNotNull(quadWriter);
            } catch (IOException e) {
                Assert.fail("Registry indicates that " + lang.getName() + " can write quads but fails to produce a triple writer when asked: " + e.getMessage());
            }
        }
    } else {
        try {
            HadoopRdfIORegistry.createQuadReader(lang);
            Assert.fail("Registry indicates that " + lang.getName() + " cannot read quads but produced a quad reader when asked (error was expected)");
        } catch (IOException e) {
        // This is expected
        }
        try {
            HadoopRdfIORegistry.createQuadWriter(lang, new StringWriter(), new Configuration(false));
            Assert.fail("Registry indicates that " + lang.getName() + " cannot write quads but produced a quad writer when asked (error was expected)");
        } catch (IOException e) {
        // This is expected
        }
    }
}

Example 23 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class TestDistinctTriples method distinct_triples_03.

@Test
public void distinct_triples_03() throws IOException {
    MapReduceDriver<LongWritable, TripleWritable, TripleWritable, NullWritable, NullWritable, TripleWritable> driver = this.getMapReduceDriver();
    Triple t = new Triple(NodeFactory.createURI("urn:s"), NodeFactory.createURI("urn:p"), NodeFactory.createLiteral("1"));
    Triple t2 = new Triple(t.getSubject(), t.getPredicate(), NodeFactory.createLiteral("2"));
    Assert.assertNotEquals(t, t2);
    TripleWritable tw = new TripleWritable(t);
    TripleWritable tw2 = new TripleWritable(t2);
    Assert.assertNotEquals(tw, tw2);
    driver.addInput(new LongWritable(1), tw);
    driver.addInput(new LongWritable(2), tw2);
    driver.addOutput(NullWritable.get(), tw);
    driver.addOutput(NullWritable.get(), tw2);
    driver.runTest(false);
}

Example 24 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class AbstractCharacteristicSetGeneratingReducerTests method characteristic_set_generating_reducer_01.

/**
     * Test basic characteristic set computation
     * 
     * @throws IOException
     */
@Test
public void characteristic_set_generating_reducer_01() throws IOException {
    MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this.getMapReduceDriver();
    T tuple = this.createTuple(1, "http://predicate");
    driver.addInput(new LongWritable(1), tuple);
    this.createSet(driver, 1, "http://predicate");
    driver.runTest(false);
}

Example 25 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class CharacteristicSetReducerTest method characteristic_set_reducer_04.

/**
     * Test characteristic set reduction
     * 
     * @throws IOException
     */
@Test
public void characteristic_set_reducer_04() throws IOException {
    MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this.getMapReduceDriver();
    this.createSet(driver, 2, 1, "http://predicate");
    this.createSet(driver, 1, 1, "http://other");
    driver.runTest(false);
    driver = getMapReduceDriver();
    createSet(driver, 2, 1, "http://predicate");
    createSet(driver, 1, 1, "http://other");
    List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
    for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
        CharacteristicSetWritable cw = pair.getFirst();
        boolean expectTwo = cw.getCharacteristics().next().getNode().get().hasURI("http://predicate");
        Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
    }
}