Examples with NullWritable org.apache.hadoop.io.NullWritable used on opensource projects

Example 96 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class AbstractCharacteristicSetGeneratingReducerTests method characteristic_set_generating_reducer_02.

/**
     * Test basic characteristic set computation
     * 
     * @throws IOException
     */
@Test
public void characteristic_set_generating_reducer_02() throws IOException {
    MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this.getMapReduceDriver();
    T tuple = this.createTuple(1, "http://predicate");
    driver.addInput(new LongWritable(1), tuple);
    driver.addInput(new LongWritable(1), tuple);
    this.createSet(driver, 1, "http://predicate");
    driver.runTest(false);
}

Example 97 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class AbstractCharacteristicSetGeneratingReducerTests method characteristic_set_generating_reducer_05.

/**
     * Test basic characteristic set computation
     * 
     * @throws IOException
     */
@Test
public void characteristic_set_generating_reducer_05() throws IOException {
    MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this.getMapReduceDriver();
    T tuple = this.createTuple(1, "http://predicate");
    driver.addInput(new LongWritable(1), tuple);
    tuple = this.createTuple(1, "http://other");
    driver.addInput(new LongWritable(2), tuple);
    tuple = this.createTuple(1, "http://third");
    driver.addInput(new LongWritable(3), tuple);
    // Single entry sets
    this.createSet(driver, 1, "http://predicate");
    this.createSet(driver, 1, "http://other");
    this.createSet(driver, 1, "http://third");
    // Two entry sets
    this.createSet(driver, 1, "http://predicate", "http://other");
    this.createSet(driver, 1, "http://predicate", "http://third");
    this.createSet(driver, 1, "http://other", "http://third");
    // Three entry sets
    this.createSet(driver, 1, "http://predicate", "http://other", "http://third");
    driver.runTest(false);
}

Example 98 with NullWritable

use of org.apache.hadoop.io.NullWritable in project jena by apache.

the class CharacteristicSetReducerTest method characteristic_set_reducer_06.

/**
     * Test characteristic set reduction
     * 
     * @throws IOException
     */
@Test
public void characteristic_set_reducer_06() throws IOException {
    MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this.getMapReduceDriver();
    this.createSet(driver, 2, 1, "http://predicate", "http://other");
    this.createSet(driver, 1, 1, "http://other");
    driver.runTest(false);
    driver = getMapReduceDriver();
    createSet(driver, 2, 1, "http://predicate", "http://other");
    createSet(driver, 1, 1, "http://other");
    List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
    for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
        CharacteristicSetWritable cw = pair.getFirst();
        boolean expectTwo = cw.hasCharacteristic("http://predicate");
        Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
    }
}

Example 99 with NullWritable

use of org.apache.hadoop.io.NullWritable in project spark-dataflow by cloudera.

the class TransformTranslator method writeAvro.

private static <T> TransformEvaluator<AvroIO.Write.Bound<T>> writeAvro() {
    return new TransformEvaluator<AvroIO.Write.Bound<T>>() {

        @Override
        public void evaluate(AvroIO.Write.Bound<T> transform, EvaluationContext context) {
            Job job;
            try {
                job = Job.getInstance();
            } catch (IOException e) {
                throw new IllegalStateException(e);
            }
            AvroJob.setOutputKeySchema(job, transform.getSchema());
            @SuppressWarnings("unchecked") JavaPairRDD<AvroKey<T>, NullWritable> last = ((JavaRDDLike<WindowedValue<T>, ?>) context.getInputRDD(transform)).map(WindowingHelpers.<T>unwindowFunction()).mapToPair(new PairFunction<T, AvroKey<T>, NullWritable>() {

                @Override
                public Tuple2<AvroKey<T>, NullWritable> call(T t) throws Exception {
                    return new Tuple2<>(new AvroKey<>(t), NullWritable.get());
                }
            });
            ShardTemplateInformation shardTemplateInfo = new ShardTemplateInformation(transform.getNumShards(), transform.getShardTemplate(), transform.getFilenamePrefix(), transform.getFilenameSuffix());
            writeHadoopFile(last, job.getConfiguration(), shardTemplateInfo, AvroKey.class, NullWritable.class, TemplatedAvroKeyOutputFormat.class);
        }
    };
}