Examples with ReduceDriver org.apache.hadoop.mrunit.mapreduce.ReduceDriver used on opensource projects

Example 1 with ReduceDriver

use of org.apache.hadoop.mrunit.mapreduce.ReduceDriver in project incubator-rya by apache.

the class CardinalityIdentityReducerTest method testCIReducerOneConstant.

@Test
public void testCIReducerOneConstant() throws InterruptedException, IOException {
    TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("object"));
    CardList cL1 = new CardList(1, 2, 3, 0, 0, 0);
    CardList cL2 = new CardList(4, 5, 6, 0, 0, 0);
    CardList cl = new CardList(5, 7, 9, 0, 0, 0);
    List<CardList> list = new ArrayList<CardList>();
    list.add(cL1);
    list.add(cL2);
    Text row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
    Mutation m1 = new Mutation(row);
    m1.put(new Text(te.getKeyPos().toString() + "subject"), new Text(cl.getcardS().toString()), new Value(new byte[0]));
    Mutation m2 = new Mutation(row);
    m2.put(new Text(te.getKeyPos().toString() + "predicate"), new Text(cl.getcardP().toString()), new Value(new byte[0]));
    Mutation m3 = new Mutation(row);
    m3.put(new Text(te.getKeyPos().toString() + "object"), new Text(cl.getcardO().toString()), new Value(new byte[0]));
    Text table = new Text("");
    new ReduceDriver<TripleEntry, CardList, Text, Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list).withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();
}

Example 2 with ReduceDriver

use of org.apache.hadoop.mrunit.mapreduce.ReduceDriver in project incubator-rya by apache.

the class CardinalityIdentityReducerTest method testJoinTwoVarsReverseOrder.

@Test
public void testJoinTwoVarsReverseOrder() throws InterruptedException, IOException {
    TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("objectpredicate"));
    CardList cL1 = new CardList(0, 0, 0, 1, 2, 3);
    CardList cL2 = new CardList(0, 0, 0, 4, 5, 6);
    CardList cl = new CardList(0, 0, 0, 5, 7, 9);
    List<CardList> list = new ArrayList<CardList>();
    list.add(cL1);
    list.add(cL2);
    Text row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
    Mutation m1 = new Mutation(row);
    m1.put(new Text("predicateobject" + "predicatesubject"), new Text(cl.getcardSP().toString()), new Value(new byte[0]));
    Mutation m2 = new Mutation(row);
    m2.put(new Text("predicateobject" + "objectpredicate"), new Text(cl.getcardPO().toString()), new Value(new byte[0]));
    Mutation m3 = new Mutation(row);
    m3.put(new Text("predicateobject" + "subjectobject"), new Text(cl.getcardSO().toString()), new Value(new byte[0]));
    Text table = new Text("");
    new ReduceDriver<TripleEntry, CardList, Text, Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list).withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();
}

Example 3 with ReduceDriver

use of org.apache.hadoop.mrunit.mapreduce.ReduceDriver in project incubator-rya by apache.

the class CardinalityIdentityReducerTest method testCIReducerTwoConstant.

@Test
public void testCIReducerTwoConstant() throws InterruptedException, IOException {
    TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text("urn:gem#pred"), new Text("subject"), new Text("predicate"), new Text("object"));
    CardList cL1 = new CardList(1, 2, 3, 0, 0, 0);
    CardList cL2 = new CardList(4, 5, 6, 0, 0, 0);
    CardList cl = new CardList(5, 7, 9, 0, 0, 0);
    List<CardList> list = new ArrayList<CardList>();
    list.add(cL1);
    list.add(cL2);
    Text row = new Text(te.getFirstPos().toString() + te.getSecondPos().toString() + DELIM + te.getFirst().toString() + DELIM + te.getSecond());
    Mutation m1 = new Mutation(row);
    m1.put(new Text(te.getKeyPos().toString() + "subject"), new Text(cl.getcardS().toString()), new Value(new byte[0]));
    Mutation m2 = new Mutation(row);
    m2.put(new Text(te.getKeyPos().toString() + "predicate"), new Text(cl.getcardP().toString()), new Value(new byte[0]));
    Mutation m3 = new Mutation(row);
    m3.put(new Text(te.getKeyPos().toString() + "object"), new Text(cl.getcardO().toString()), new Value(new byte[0]));
    Text table = new Text("");
    new ReduceDriver<TripleEntry, CardList, Text, Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list).withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();
}

Example 4 with ReduceDriver

use of org.apache.hadoop.mrunit.mapreduce.ReduceDriver in project incubator-rya by apache.

the class CardinalityIdentityReducerTest method testJoinTwoVars.

@Test
public void testJoinTwoVars() throws InterruptedException, IOException {
    TripleEntry te = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("predicateobject"));
    CardList cL1 = new CardList(0, 0, 0, 1, 2, 3);
    CardList cL2 = new CardList(0, 0, 0, 4, 5, 6);
    CardList cl = new CardList(0, 0, 0, 5, 7, 9);
    List<CardList> list = new ArrayList<CardList>();
    list.add(cL1);
    list.add(cL2);
    Text row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
    Mutation m1 = new Mutation(row);
    m1.put(new Text(te.getKeyPos().toString() + "subjectpredicate"), new Text(cl.getcardSP().toString()), new Value(new byte[0]));
    Mutation m2 = new Mutation(row);
    m2.put(new Text(te.getKeyPos().toString() + "predicateobject"), new Text(cl.getcardPO().toString()), new Value(new byte[0]));
    Mutation m3 = new Mutation(row);
    m3.put(new Text(te.getKeyPos().toString() + "objectsubject"), new Text(cl.getcardSO().toString()), new Value(new byte[0]));
    Text table = new Text("");
    new ReduceDriver<TripleEntry, CardList, Text, Mutation>().withReducer(new JoinSelectStatisticsSum.CardinalityIdentityReducer()).withInput(te, list).withOutput(table, m1).withOutput(table, m2).withOutput(table, m3).runTest();
}

Example 5 with ReduceDriver

use of org.apache.hadoop.mrunit.mapreduce.ReduceDriver in project incubator-rya by apache.

the class ForwardChainTest method testTransitiveChain.

/**
 * MultipleOutputs support is minimal, so we have to check each map/reduce
 * step explicitly
 */
@Test
public void testTransitiveChain() throws Exception {
    int max = 8;
    int n = 4;
    URI prop = TestUtils.uri("subOrganizationOf");
    Map<Integer, Map<Integer, Pair<Fact, NullWritable>>> connections = new HashMap<>();
    for (int i = 0; i <= max; i++) {
        connections.put(i, new HashMap<Integer, Pair<Fact, NullWritable>>());
    }
    // Initial input: make a chain from org0 to org8
    for (int i = 0; i < max; i++) {
        URI orgI = TestUtils.uri("org" + i);
        URI orgJ = TestUtils.uri("org" + (i + 1));
        Fact triple = new Fact(orgI, prop, orgJ);
        connections.get(i).put(i + 1, new Pair<>(triple, NullWritable.get()));
    }
    for (int i = 1; i <= n; i++) {
        // Map:
        MapDriver<Fact, NullWritable, ResourceWritable, Fact> mDriver = new MapDriver<>();
        mDriver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, i);
        mDriver.setMapper(new ForwardChain.FileMapper(schema));
        for (int j : connections.keySet()) {
            for (int k : connections.get(j).keySet()) {
                mDriver.addInput(connections.get(j).get(k));
            }
        }
        List<Pair<ResourceWritable, Fact>> mapped = mDriver.run();
        // Convert data for reduce phase:
        ReduceFeeder<ResourceWritable, Fact> feeder = new ReduceFeeder<>(mDriver.getConfiguration());
        List<KeyValueReuseList<ResourceWritable, Fact>> intermediate = feeder.sortAndGroup(mapped, new ResourceWritable.SecondaryComparator(), new ResourceWritable.PrimaryComparator());
        // Reduce, and compare to expected output:
        ReduceDriver<ResourceWritable, Fact, Fact, NullWritable> rDriver = new ReduceDriver<>();
        rDriver.getConfiguration().setInt(MRReasoningUtils.STEP_PROP, i);
        rDriver.setReducer(new ForwardChain.ReasoningReducer(schema));
        rDriver.addAllElements(intermediate);
        int maxSpan = (int) Math.pow(2, i);
        int minSpan = (maxSpan / 2) + 1;
        // For each j, build all paths starting with j:
        for (int j = 0; j < max; j++) {
            // This includes any path of length k for appropriate k:
            for (int k = minSpan; k <= maxSpan && j + k <= max; k++) {
                int middle = j + minSpan - 1;
                URI left = TestUtils.uri("org" + j);
                URI right = TestUtils.uri("org" + (j + k));
                Fact triple = new Fact(left, prop, right, i, OwlRule.PRP_TRP, TestUtils.uri("org" + middle));
                triple.addSource(connections.get(j).get(middle).getFirst());
                triple.addSource(connections.get(middle).get(j + k).getFirst());
                Pair<Fact, NullWritable> expected = new Pair<>(triple, NullWritable.get());
                connections.get(j).put(j + k, expected);
                rDriver.addMultiOutput("intermediate", expected);
            }
        }
        rDriver.runTest();
    }
}