Examples with KeyValuesReader org.apache.tez.runtime.library.api.KeyValuesReader used on opensource projects

Example 11 with KeyValuesReader

use of org.apache.tez.runtime.library.api.KeyValuesReader in project tez by apache.

the class TestSortedGroupedMergedInput method testOrdering.

@Test(timeout = 5000)
public void testOrdering() throws Exception {
    SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 2, 4 }, new int[][] { { 2, 2 }, { 4, 4 } });
    SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
    SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 4, 5, 6, 7 }, new int[][] { { 4, 4 }, { 5, 5 }, { 6, 6 }, { 7, 7 } });
    SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
    SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
    SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
    List<Input> sInputs = new LinkedList<Input>();
    sInputs.add(sInput1);
    sInputs.add(sInput2);
    sInputs.add(sInput3);
    OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
    KeyValuesReader kvsReader = input.getReader();
    int keyCount = 0;
    while (kvsReader.next()) {
        keyCount++;
        Integer key = (Integer) kvsReader.getCurrentKey();
        assertEquals(Integer.valueOf(keyCount), key);
        Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
        int valCount = 0;
        while (valuesIter.hasNext()) {
            valCount++;
            Integer val = (Integer) valuesIter.next();
            assertEquals(Integer.valueOf(keyCount), val);
        }
        if (keyCount == 1) {
            assertEquals(2, valCount);
        } else if (keyCount == 2) {
            assertEquals(4, valCount);
        } else if (keyCount == 3) {
            assertEquals(2, valCount);
        } else if (keyCount == 4) {
            assertEquals(4, valCount);
        } else if (keyCount == 5 || keyCount == 6 || keyCount == 7) {
            assertEquals(2, valCount);
        } else {
            fail("Unexpected key");
        }
    }
    getNextFromFinishedReader(kvsReader);
}

Example 12 with KeyValuesReader

use of org.apache.tez.runtime.library.api.KeyValuesReader in project tez by apache.

the class TestSortedGroupedMergedInput method testEmptySources.

@Test(timeout = 5000)
public void testEmptySources() throws Exception {
    SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] {}, new int[][] {});
    SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] {}, new int[][] {});
    SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] {}, new int[][] {});
    SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
    SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
    SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
    List<Input> sInputs = new LinkedList<Input>();
    sInputs.add(sInput1);
    sInputs.add(sInput2);
    sInputs.add(sInput3);
    OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
    KeyValuesReader kvsReader = input.getReader();
    assertTrue(kvsReader.next() == false);
    getNextFromFinishedReader(kvsReader);
}

Example 13 with KeyValuesReader

use of org.apache.tez.runtime.library.api.KeyValuesReader in project tez by apache.

the class OrderedGroupedKVInput method getReader.

/**
 * Get a KVReader for the Input.</p> This method will block until the input is
 * ready - i.e. the copy and merge stages are complete. Users can use the
 * isInputReady method to check if the input is ready, which gives an
 * indication of whether this method will block or not.
 *
 * NOTE: All values for the current K-V pair must be read prior to invoking
 * moveToNext. Once moveToNext() is called, the valueIterator from the
 * previous K-V pair will throw an Exception
 *
 * @return a KVReader over the sorted input.
 * @throws {@link IOInterruptedException} if IO was performing a blocking operation and was interrupted
 */
@Override
public KeyValuesReader getReader() throws IOException, TezException {
    // Cannot synchronize entire method since this is called form user code and can block.
    TezRawKeyValueIterator rawIterLocal;
    synchronized (this) {
        rawIterLocal = rawIter;
        if (getNumPhysicalInputs() == 0) {
            return new KeyValuesReader() {

                @Override
                public boolean next() throws IOException {
                    getContext().notifyProgress();
                    hasCompletedProcessing();
                    completedProcessing = true;
                    return false;
                }

                @Override
                public Object getCurrentKey() throws IOException {
                    throw new RuntimeException("No data available in Input");
                }

                @Override
                public Iterable<Object> getCurrentValues() throws IOException {
                    throw new RuntimeException("No data available in Input");
                }
            };
        }
    }
    if (rawIterLocal == null) {
        try {
            waitForInputReady();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new IOInterruptedException("Interrupted while waiting for input ready", e);
        }
    }
    @SuppressWarnings("rawtypes") ValuesIterator valuesIter = null;
    synchronized (this) {
        valuesIter = vIter;
    }
    return new OrderedGroupedKeyValuesReader(valuesIter, getContext());
}