Examples with MapDriver org.apache.hadoop.mrunit.mapreduce.MapDriver used on opensource projects

Example 1 with MapDriver

use of org.apache.hadoop.mrunit.mapreduce.MapDriver in project incubator-rya by apache.

the class JoinSelectMapperTest method testOutput.

@Test
public void testOutput() throws TripleRowResolverException, IOException {
    RyaStatement rya = new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1"));
    Text s = new Text(rya.getSubject().getData());
    Text p = new Text(rya.getPredicate().getData());
    Text o = new Text(rya.getObject().getData());
    Text sp = new Text(rya.getSubject().getData() + DELIM + rya.getPredicate().getData());
    Text so = new Text(rya.getSubject().getData() + DELIM + rya.getObject().getData());
    Text po = new Text(rya.getPredicate().getData() + DELIM + rya.getObject().getData());
    Text ps = new Text(rya.getPredicate().getData() + DELIM + rya.getSubject().getData());
    Text op = new Text(rya.getObject().getData() + DELIM + rya.getPredicate().getData());
    Text os = new Text(rya.getObject().getData() + DELIM + rya.getSubject().getData());
    TripleEntry t1 = new TripleEntry(s, p, new Text("subject"), new Text("predicate"), new Text("object"));
    TripleEntry t2 = new TripleEntry(p, o, new Text("predicate"), new Text("object"), new Text("subject"));
    TripleEntry t3 = new TripleEntry(o, s, new Text("object"), new Text("subject"), new Text("predicate"));
    TripleEntry t4 = new TripleEntry(o, new Text(""), new Text("object"), new Text(""), new Text("subjectpredicate"));
    TripleEntry t5 = new TripleEntry(p, new Text(""), new Text("predicate"), new Text(""), new Text("objectsubject"));
    TripleEntry t6 = new TripleEntry(s, new Text(""), new Text("subject"), new Text(""), new Text("predicateobject"));
    TripleEntry t7 = new TripleEntry(s, new Text(""), new Text("subject"), new Text(""), new Text("objectpredicate"));
    TripleEntry t8 = new TripleEntry(p, new Text(""), new Text("predicate"), new Text(""), new Text("subjectobject"));
    TripleEntry t9 = new TripleEntry(o, new Text(""), new Text("object"), new Text(""), new Text("predicatesubject"));
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT, TripleRow> map = trr.serialize(rya);
    System.out.println(map);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    System.out.println("Triple row is" + tr);
    System.out.println("ColumnV is " + tr.getTimestamp());
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(), tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    new MapDriver<Key, Value, CompositeType, TripleCard>().withMapper(new JoinSelectSpoTableOutput.JoinSelectMapper()).withInput(key, val).withOutput(new CompositeType(o, new IntWritable(2)), new TripleCard(t1)).withOutput(new CompositeType(s, new IntWritable(2)), new TripleCard(t2)).withOutput(new CompositeType(p, new IntWritable(2)), new TripleCard(t3)).withOutput(new CompositeType(po, new IntWritable(2)), new TripleCard(t6)).withOutput(new CompositeType(so, new IntWritable(2)), new TripleCard(t5)).withOutput(new CompositeType(sp, new IntWritable(2)), new TripleCard(t4)).withOutput(new CompositeType(op, new IntWritable(2)), new TripleCard(t7)).withOutput(new CompositeType(os, new IntWritable(2)), new TripleCard(t8)).withOutput(new CompositeType(ps, new IntWritable(2)), new TripleCard(t9)).runTest();
}

Example 2 with MapDriver

use of org.apache.hadoop.mrunit.mapreduce.MapDriver in project incubator-rya by apache.

the class DuplicateEliminationTest method testRdfMapperOutput.

@Test
public void testRdfMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    RyaStatementWritable rsw = new RyaStatementWritable();
    rsw.setRyaStatement(rya);
    LongWritable l = new LongWritable();
    new MapDriver<LongWritable, RyaStatementWritable, Fact, Derivation>().withMapper(new DuplicateElimination.DuplicateRdfMapper()).withInput(l, rsw).withOutput(X_SUB_Y, X_SUB_Y.getDerivation()).runTest();
}

Example 3 with MapDriver

use of org.apache.hadoop.mrunit.mapreduce.MapDriver in project incubator-rya by apache.

the class ForwardChainTest method testTransitiveChain.

/**
 * MultipleOutputs support is minimal, so we have to check each map/reduce
 * step explicitly
 */
@Test
public void testTransitiveChain() throws Exception {
    int max = 8;
    int n = 4;
    URI prop = TestUtils.uri("subOrganizationOf");
    Map<Integer, Map<Integer, Pair<Fact, NullWritable>>> connections = new HashMap<>();
    for (int i = 0; i <= max; i++) {
        connections.put(i, new HashMap<Integer, Pair<Fact, NullWritable>>());
    }
    // Initial input: make a chain from org0 to org8
    for (int i = 0; i < max; i++) {
        URI orgI = TestUtils.uri("org" + i);
        URI orgJ = TestUtils.uri("org" + (i + 1));
        Fact triple = new Fact(orgI, prop, orgJ);
        connections.get(i).put(i + 1, new Pair<>(triple, NullWritable.get()));
    }
    for (int i = 1; i <= n; i++) {
        // Map:
        MapDriver<Fact, NullWritable, ResourceWritable, Fact> mDriver = new MapDriver<>();
        mDriver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, i);
        mDriver.setMapper(new ForwardChain.FileMapper(schema));
        for (int j : connections.keySet()) {
            for (int k : connections.get(j).keySet()) {
                mDriver.addInput(connections.get(j).get(k));
            }
        }
        List<Pair<ResourceWritable, Fact>> mapped = mDriver.run();
        // Convert data for reduce phase:
        ReduceFeeder<ResourceWritable, Fact> feeder = new ReduceFeeder<>(mDriver.getConfiguration());
        List<KeyValueReuseList<ResourceWritable, Fact>> intermediate = feeder.sortAndGroup(mapped, new ResourceWritable.SecondaryComparator(), new ResourceWritable.PrimaryComparator());
        // Reduce, and compare to expected output:
        ReduceDriver<ResourceWritable, Fact, Fact, NullWritable> rDriver = new ReduceDriver<>();
        rDriver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, i);
        rDriver.setReducer(new ForwardChain.ReasoningReducer(schema));
        rDriver.addAllElements(intermediate);
        int maxSpan = (int) Math.pow(2, i);
        int minSpan = (maxSpan / 2) + 1;
        // For each j, build all paths starting with j:
        for (int j = 0; j < max; j++) {
            // This includes any path of length k for appropriate k:
            for (int k = minSpan; k <= maxSpan && j + k <= max; k++) {
                int middle = j + minSpan - 1;
                URI left = TestUtils.uri("org" + j);
                URI right = TestUtils.uri("org" + (j + k));
                Fact triple = new Fact(left, prop, right, i, OwlRule.PRP_TRP, TestUtils.uri("org" + middle));
                triple.addSource(connections.get(j).get(middle).getFirst());
                triple.addSource(connections.get(middle).get(j + k).getFirst());
                Pair<Fact, NullWritable> expected = new Pair<>(triple, NullWritable.get());
                connections.get(j).put(j + k, expected);
                rDriver.addMultiOutput("intermediate", expected);
            }
        }
        rDriver.runTest();
    }
}

Example 4 with MapDriver

use of org.apache.hadoop.mrunit.mapreduce.MapDriver in project incubator-rya by apache.

the class CardinalityMapperTest method testOutput.

@Test
public void testOutput() throws InterruptedException, IOException {
    String s = "urn:gem:etype#1234";
    String p = "urn:gem#pred";
    Text t1 = new Text(TripleValueType.subject.name() + DELIM + s + DELIM + 1);
    Text t2 = new Text(TripleValueType.predicate.name() + DELIM + p + DELIM + 2);
    Text t3 = new Text(TripleValueType.subjectpredicate.name() + DELIM + s + DELIM + p + DELIM + 3);
    byte[] b = new byte[0];
    byte[] c = "25".getBytes();
    byte[] d = "47".getBytes();
    byte[] e = "15".getBytes();
    Key key1 = new Key(t1.getBytes(), b, b, b, 1);
    Key key2 = new Key(t2.getBytes(), b, b, b, 1);
    Key key3 = new Key(t3.getBytes(), b, b, b, 1);
    Value val1 = new Value(c);
    Value val2 = new Value(d);
    Value val3 = new Value(e);
    // System.out.println("Keys are " + key1 + " and " + key2);
    new MapDriver<Key, Value, CompositeType, TripleCard>().withMapper(new JoinSelectProspectOutput.CardinalityMapper()).withInput(key1, val1).withInput(key2, val2).withInput(key3, val3).withOutput(new CompositeType(s, 1), new TripleCard(new CardinalityType(25, "subject", 1))).withOutput(new CompositeType(p, 1), new TripleCard(new CardinalityType(47, "predicate", 2))).withOutput(new CompositeType(s + DELIM + p, 1), new TripleCard(new CardinalityType(15, "subjectpredicate", 3))).runTest();
}

Example 5 with MapDriver

use of org.apache.hadoop.mrunit.mapreduce.MapDriver in project incubator-rya by apache.

the class JoinSelectProspectOutputTest method testOutput.

@Test
public void testOutput() throws InterruptedException, IOException {
    String s = "urn:gem:etype#1234";
    String p = "urn:gem#pred";
    String ts = "798497748386999999";
    Text t1 = new Text(TripleValueType.subject.name() + DELIM + s + DELIM + 1);
    Text t2 = new Text(TripleValueType.predicate.name() + DELIM + p + DELIM + 2);
    Text t3 = new Text(TripleValueType.subjectpredicate.name() + DELIM + s + DELIM + p + DELIM + ts);
    byte[] b = new byte[0];
    byte[] c = "25".getBytes();
    byte[] d = "47".getBytes();
    byte[] e = "15".getBytes();
    Key key1 = new Key(t1.getBytes(), b, b, b, 1);
    Key key2 = new Key(t2.getBytes(), b, b, b, 1);
    Key key3 = new Key(t3.getBytes(), b, b, b, 1);
    Value val1 = new Value(c);
    Value val2 = new Value(d);
    Value val3 = new Value(e);
    // System.out.println("Keys are " + key1 + " and " + key2);
    new MapDriver<Key, Value, CompositeType, TripleCard>().withMapper(new JoinSelectProspectOutput.CardinalityMapper()).withInput(key1, val1).withInput(key2, val2).withInput(key3, val3).withOutput(new CompositeType(s, 1), new TripleCard(new CardinalityType(25, "subject", 1))).withOutput(new CompositeType(p, 1), new TripleCard(new CardinalityType(47, "predicate", 2))).withOutput(new CompositeType(s + DELIM + p, 1), new TripleCard(new CardinalityType(15, "subjectpredicate", Long.parseLong(ts)))).runTest();
}