Examples with MergedLogicalInput org.apache.tez.runtime.api.MergedLogicalInput used on opensource projects

Example 1 with MergedLogicalInput

use of org.apache.tez.runtime.api.MergedLogicalInput in project tez by apache.

the class LogicalIOProcessorRuntimeTask method initializeGroupInputs.

private void initializeGroupInputs() throws TezException {
    if (groupInputSpecs != null && !groupInputSpecs.isEmpty()) {
        groupInputsMap = new ConcurrentHashMap<String, MergedLogicalInput>(groupInputSpecs.size());
        for (GroupInputSpec groupInputSpec : groupInputSpecs) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Initializing GroupInput using GroupInputSpec: " + groupInputSpec);
            }
            MergedInputContext mergedInputContext = new TezMergedInputContextImpl(groupInputSpec.getMergedInputDescriptor().getUserPayload(), groupInputSpec.getGroupName(), groupInputsMap, inputReadyTracker, localDirs, this);
            List<Input> inputs = Lists.newArrayListWithCapacity(groupInputSpec.getGroupVertices().size());
            for (String groupVertex : groupInputSpec.getGroupVertices()) {
                inputs.add(inputsMap.get(groupVertex));
            }
            MergedLogicalInput groupInput = (MergedLogicalInput) createMergedInput(groupInputSpec.getMergedInputDescriptor(), mergedInputContext, inputs);
            groupInputsMap.put(groupInputSpec.getGroupName(), groupInput);
        }
    }
}

Example 2 with MergedLogicalInput

use of org.apache.tez.runtime.api.MergedLogicalInput in project tez by apache.

the class InputReadyTracker method setGroupedInputs.

public void setGroupedInputs(Collection<MergedLogicalInput> inputGroups) {
    lock.lock();
    try {
        if (inputGroups != null) {
            inputToGroupMap = Maps.newConcurrentMap();
            for (MergedLogicalInput mergedInput : inputGroups) {
                for (Input dest : mergedInput.getInputs()) {
                    // Check already ready Inputs - may have become ready during initialize
                    if (readyInputs.containsKey(dest)) {
                        mergedInput.setConstituentInputIsReady(dest);
                    }
                    List<MergedLogicalInput> mergedList = inputToGroupMap.get(dest);
                    if (mergedList == null) {
                        mergedList = Lists.newArrayList();
                        inputToGroupMap.put(dest, mergedList);
                    }
                    mergedList.add(mergedInput);
                }
            }
        }
    } finally {
        lock.unlock();
    }
}

Example 3 with MergedLogicalInput

use of org.apache.tez.runtime.api.MergedLogicalInput in project tez by apache.

the class TestInputReadyTracker method testGrouped.

@Test(timeout = 20000)
public void testGrouped() throws InterruptedException {
    InputReadyTracker inputReadyTracker = new InputReadyTracker();
    ImmediatelyReadyInputForTest input1 = new ImmediatelyReadyInputForTest(inputReadyTracker);
    ControlledReadyInputForTest input2 = new ControlledReadyInputForTest(inputReadyTracker);
    ImmediatelyReadyInputForTest input3 = new ImmediatelyReadyInputForTest(inputReadyTracker);
    ControlledReadyInputForTest input4 = new ControlledReadyInputForTest(inputReadyTracker);
    List<Input> group1Inputs = new ArrayList<Input>();
    group1Inputs.add(input1);
    group1Inputs.add(input2);
    List<Input> group2Inputs = new ArrayList<Input>();
    group2Inputs.add(input3);
    group2Inputs.add(input4);
    Map<String, MergedLogicalInput> mergedInputMap = new HashMap<String, MergedLogicalInput>();
    MergedInputContext mergedInputContext1 = new TezMergedInputContextImpl(null, "group1", mergedInputMap, inputReadyTracker, null, null);
    MergedInputContext mergedInputContext2 = new TezMergedInputContextImpl(null, "group2", mergedInputMap, inputReadyTracker, null, null);
    AnyOneMergedInputForTest group1 = new AnyOneMergedInputForTest(mergedInputContext1, group1Inputs);
    AllMergedInputForTest group2 = new AllMergedInputForTest(mergedInputContext2, group2Inputs);
    mergedInputMap.put("group1", group1);
    mergedInputMap.put("group2", group2);
    // Register groups with tracker
    List<MergedLogicalInput> groups = Lists.newArrayList(group1, group2);
    inputReadyTracker.setGroupedInputs(groups);
    // Test for simple inputs
    List<Input> requestList;
    long startTime = 0l;
    long readyTime = 0l;
    requestList = new ArrayList<Input>();
    requestList.add(group1);
    Input readyInput = inputReadyTracker.waitForAnyInputReady(requestList);
    assertTrue(group1.isReady);
    assertTrue(input1.isReady);
    assertFalse(input2.isReady);
    assertEquals(group1, readyInput);
    requestList = new ArrayList<Input>();
    requestList.add(group2);
    startTime = System.nanoTime();
    setDelayedInputReady(input4);
    inputReadyTracker.waitForAllInputsReady(requestList);
    readyTime = System.nanoTime();
    // Should have moved into ready state - only happens when the setReady function is invoked.
    // Ensure the method returned only after the specific Input was told it is ready
    assertTrue(group2.isReady);
    assertTrue(input3.isReady);
    assertTrue(input4.isReady);
    assertTrue(readyTime >= startTime + SLEEP_TIME);
}