Examples with MergedInputContext org.apache.tez.runtime.api.MergedInputContext used on opensource projects

Example 1 with MergedInputContext

use of org.apache.tez.runtime.api.MergedInputContext in project tez by apache.

the class TestSortedGroupedMergedInput method testSimpleConcatenatedMergedKeyValuesInput.

@Test(timeout = 5000)
public void testSimpleConcatenatedMergedKeyValuesInput() throws Exception {
    SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
    SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
    SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
    List<Input> sInputs = new LinkedList<Input>();
    sInputs.add(sInput1);
    sInputs.add(sInput2);
    sInputs.add(sInput3);
    MergedInputContext mockContext = createMergedInputContext();
    ConcatenatedMergedKeyValuesInput input = new ConcatenatedMergedKeyValuesInput(mockContext, sInputs);
    KeyValuesReader kvsReader = input.getReader();
    int keyCount = 0;
    while (kvsReader.next()) {
        keyCount++;
        Integer key = (Integer) kvsReader.getCurrentKey();
        Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
        int valCount = 0;
        while (valuesIter.hasNext()) {
            valCount++;
            Integer val = (Integer) valuesIter.next();
        }
        assertEquals(2, valCount);
    }
    assertEquals(9, keyCount);
    // one for each reader change and one to exit
    verify(mockContext, times(4)).notifyProgress();
    getNextFromFinishedReader(kvsReader);
}

Example 2 with MergedInputContext

use of org.apache.tez.runtime.api.MergedInputContext in project tez by apache.

the class TestSortedGroupedMergedInput method testSimple.

@Test(timeout = 5000)
public void testSimple() throws Exception {
    SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
    SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
    SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
    List<Input> sInputs = new LinkedList<Input>();
    sInputs.add(sInput1);
    sInputs.add(sInput2);
    sInputs.add(sInput3);
    MergedInputContext mockContext = createMergedInputContext();
    OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(mockContext, sInputs);
    KeyValuesReader kvsReader = input.getReader();
    int keyCount = 0;
    while (kvsReader.next()) {
        keyCount++;
        Integer key = (Integer) kvsReader.getCurrentKey();
        assertEquals(Integer.valueOf(keyCount), key);
        Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
        int valCount = 0;
        while (valuesIter.hasNext()) {
            valCount++;
            Integer val = (Integer) valuesIter.next();
            assertEquals(Integer.valueOf(keyCount), val);
        }
        assertEquals(6, valCount);
    }
    // one for each reader change and one to exit
    verify(mockContext, times(4)).notifyProgress();
    getNextFromFinishedReader(kvsReader);
}

Example 3 with MergedInputContext

use of org.apache.tez.runtime.api.MergedInputContext in project tez by apache.

the class TestSortedGroupedMergedInput method testSimpleConcatenatedMergedKeyValueInput.

@Test(timeout = 5000)
public void testSimpleConcatenatedMergedKeyValueInput() throws Exception {
    DummyInput sInput1 = new DummyInput(10);
    DummyInput sInput2 = new DummyInput(10);
    DummyInput sInput3 = new DummyInput(10);
    List<Input> sInputs = new LinkedList<Input>();
    sInputs.add(sInput1);
    sInputs.add(sInput2);
    sInputs.add(sInput3);
    MergedInputContext mockContext = createMergedInputContext();
    ConcatenatedMergedKeyValueInput input = new ConcatenatedMergedKeyValueInput(mockContext, sInputs);
    KeyValueReader kvReader = input.getReader();
    int keyCount = 0;
    while (kvReader.next()) {
        keyCount++;
        Integer key = (Integer) kvReader.getCurrentKey();
        Integer value = (Integer) kvReader.getCurrentValue();
    }
    assertTrue(keyCount == 30);
    // one for each reader change and one to exit
    verify(mockContext, times(4)).notifyProgress();
    getNextFromFinishedReader(kvReader);
}

Example 4 with MergedInputContext

use of org.apache.tez.runtime.api.MergedInputContext in project tez by apache.

the class LogicalIOProcessorRuntimeTask method initializeGroupInputs.

private void initializeGroupInputs() throws TezException {
    if (groupInputSpecs != null && !groupInputSpecs.isEmpty()) {
        groupInputsMap = new ConcurrentHashMap<String, MergedLogicalInput>(groupInputSpecs.size());
        for (GroupInputSpec groupInputSpec : groupInputSpecs) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Initializing GroupInput using GroupInputSpec: " + groupInputSpec);
            }
            MergedInputContext mergedInputContext = new TezMergedInputContextImpl(groupInputSpec.getMergedInputDescriptor().getUserPayload(), groupInputSpec.getGroupName(), groupInputsMap, inputReadyTracker, localDirs, this);
            List<Input> inputs = Lists.newArrayListWithCapacity(groupInputSpec.getGroupVertices().size());
            for (String groupVertex : groupInputSpec.getGroupVertices()) {
                inputs.add(inputsMap.get(groupVertex));
            }
            MergedLogicalInput groupInput = (MergedLogicalInput) createMergedInput(groupInputSpec.getMergedInputDescriptor(), mergedInputContext, inputs);
            groupInputsMap.put(groupInputSpec.getGroupName(), groupInput);
        }
    }
}

Example 5 with MergedInputContext

use of org.apache.tez.runtime.api.MergedInputContext in project tez by apache.

the class TestInputReadyTracker method testGrouped.

@Test(timeout = 20000)
public void testGrouped() throws InterruptedException {
    InputReadyTracker inputReadyTracker = new InputReadyTracker();
    ImmediatelyReadyInputForTest input1 = new ImmediatelyReadyInputForTest(inputReadyTracker);
    ControlledReadyInputForTest input2 = new ControlledReadyInputForTest(inputReadyTracker);
    ImmediatelyReadyInputForTest input3 = new ImmediatelyReadyInputForTest(inputReadyTracker);
    ControlledReadyInputForTest input4 = new ControlledReadyInputForTest(inputReadyTracker);
    List<Input> group1Inputs = new ArrayList<Input>();
    group1Inputs.add(input1);
    group1Inputs.add(input2);
    List<Input> group2Inputs = new ArrayList<Input>();
    group2Inputs.add(input3);
    group2Inputs.add(input4);
    Map<String, MergedLogicalInput> mergedInputMap = new HashMap<String, MergedLogicalInput>();
    MergedInputContext mergedInputContext1 = new TezMergedInputContextImpl(null, "group1", mergedInputMap, inputReadyTracker, null, null);
    MergedInputContext mergedInputContext2 = new TezMergedInputContextImpl(null, "group2", mergedInputMap, inputReadyTracker, null, null);
    AnyOneMergedInputForTest group1 = new AnyOneMergedInputForTest(mergedInputContext1, group1Inputs);
    AllMergedInputForTest group2 = new AllMergedInputForTest(mergedInputContext2, group2Inputs);
    mergedInputMap.put("group1", group1);
    mergedInputMap.put("group2", group2);
    // Register groups with tracker
    List<MergedLogicalInput> groups = Lists.newArrayList(group1, group2);
    inputReadyTracker.setGroupedInputs(groups);
    // Test for simple inputs
    List<Input> requestList;
    long startTime = 0l;
    long readyTime = 0l;
    requestList = new ArrayList<Input>();
    requestList.add(group1);
    Input readyInput = inputReadyTracker.waitForAnyInputReady(requestList);
    assertTrue(group1.isReady);
    assertTrue(input1.isReady);
    assertFalse(input2.isReady);
    assertEquals(group1, readyInput);
    requestList = new ArrayList<Input>();
    requestList.add(group2);
    startTime = System.nanoTime();
    setDelayedInputReady(input4);
    inputReadyTracker.waitForAllInputsReady(requestList);
    readyTime = System.nanoTime();
    // Should have moved into ready state - only happens when the setReady function is invoked.
    // Ensure the method returned only after the specific Input was told it is ready
    assertTrue(group2.isReady);
    assertTrue(input3.isReady);
    assertTrue(input4.isReady);
    assertTrue(readyTime >= startTime + SLEEP_TIME);
}