use of org.apache.tez.runtime.api.Input in project tez by apache.
the class TestSortedGroupedMergedInput method testSkippedKey3.
// Reads all values for a key, but doesn't trigger the last hasNext() call.
@Test(timeout = 5000)
public void testSkippedKey3() throws Exception {
SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3, 4 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 } });
SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3, 4 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 } });
SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3, 4 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 } });
SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
List<Input> sInputs = new LinkedList<Input>();
sInputs.add(sInput1);
sInputs.add(sInput2);
sInputs.add(sInput3);
OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
keyCount++;
if (keyCount == 2 || keyCount == 3) {
continue;
}
Integer key = (Integer) kvsReader.getCurrentKey();
assertEquals(Integer.valueOf(keyCount), key);
Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
int valCount = 0;
while (valuesIter.hasNext()) {
valCount++;
Integer val = (Integer) valuesIter.next();
assertEquals(Integer.valueOf(keyCount), val);
if (keyCount == 1 && valCount == 6) {
// Avoid last hasNext on iterator
break;
}
}
assertEquals(6, valCount);
}
getNextFromFinishedReader(kvsReader);
}
use of org.apache.tez.runtime.api.Input in project tez by apache.
the class TestSortedGroupedMergedInput method testSimpleConcatenatedMergedKeyValuesInput.
@Test(timeout = 5000)
public void testSimpleConcatenatedMergedKeyValuesInput() throws Exception {
SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
List<Input> sInputs = new LinkedList<Input>();
sInputs.add(sInput1);
sInputs.add(sInput2);
sInputs.add(sInput3);
MergedInputContext mockContext = createMergedInputContext();
ConcatenatedMergedKeyValuesInput input = new ConcatenatedMergedKeyValuesInput(mockContext, sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
keyCount++;
Integer key = (Integer) kvsReader.getCurrentKey();
Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
int valCount = 0;
while (valuesIter.hasNext()) {
valCount++;
Integer val = (Integer) valuesIter.next();
}
assertEquals(2, valCount);
}
assertEquals(9, keyCount);
// one for each reader change and one to exit
verify(mockContext, times(4)).notifyProgress();
getNextFromFinishedReader(kvsReader);
}
use of org.apache.tez.runtime.api.Input in project tez by apache.
the class TestSortedGroupedMergedInput method testSimple.
@Test(timeout = 5000)
public void testSimple() throws Exception {
SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
List<Input> sInputs = new LinkedList<Input>();
sInputs.add(sInput1);
sInputs.add(sInput2);
sInputs.add(sInput3);
MergedInputContext mockContext = createMergedInputContext();
OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(mockContext, sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
keyCount++;
Integer key = (Integer) kvsReader.getCurrentKey();
assertEquals(Integer.valueOf(keyCount), key);
Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
int valCount = 0;
while (valuesIter.hasNext()) {
valCount++;
Integer val = (Integer) valuesIter.next();
assertEquals(Integer.valueOf(keyCount), val);
}
assertEquals(6, valCount);
}
// one for each reader change and one to exit
verify(mockContext, times(4)).notifyProgress();
getNextFromFinishedReader(kvsReader);
}
use of org.apache.tez.runtime.api.Input in project tez by apache.
the class TestSortedGroupedMergedInput method testSimpleConcatenatedMergedKeyValueInput.
@Test(timeout = 5000)
public void testSimpleConcatenatedMergedKeyValueInput() throws Exception {
DummyInput sInput1 = new DummyInput(10);
DummyInput sInput2 = new DummyInput(10);
DummyInput sInput3 = new DummyInput(10);
List<Input> sInputs = new LinkedList<Input>();
sInputs.add(sInput1);
sInputs.add(sInput2);
sInputs.add(sInput3);
MergedInputContext mockContext = createMergedInputContext();
ConcatenatedMergedKeyValueInput input = new ConcatenatedMergedKeyValueInput(mockContext, sInputs);
KeyValueReader kvReader = input.getReader();
int keyCount = 0;
while (kvReader.next()) {
keyCount++;
Integer key = (Integer) kvReader.getCurrentKey();
Integer value = (Integer) kvReader.getCurrentValue();
}
assertTrue(keyCount == 30);
// one for each reader change and one to exit
verify(mockContext, times(4)).notifyProgress();
getNextFromFinishedReader(kvReader);
}
use of org.apache.tez.runtime.api.Input in project tez by apache.
the class TestSortedGroupedMergedInput method testSkippedKey2.
@Test(timeout = 5000)
public void testSkippedKey2() throws Exception {
SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 2, 4 }, new int[][] { { 2, 2 }, { 4, 4 } });
SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } });
SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 4, 5, 6, 7 }, new int[][] { { 4, 4 }, { 5, 5 }, { 6, 6 }, { 7, 7 } });
SortedTestInput sInput1 = new SortedTestInput(kvsReader1);
SortedTestInput sInput2 = new SortedTestInput(kvsReader2);
SortedTestInput sInput3 = new SortedTestInput(kvsReader3);
List<Input> sInputs = new LinkedList<Input>();
sInputs.add(sInput1);
sInputs.add(sInput2);
sInputs.add(sInput3);
OrderedGroupedMergedKVInput input = new OrderedGroupedMergedKVInput(createMergedInputContext(), sInputs);
KeyValuesReader kvsReader = input.getReader();
int keyCount = 0;
while (kvsReader.next()) {
keyCount++;
if (keyCount == 4) {
continue;
}
Integer key = (Integer) kvsReader.getCurrentKey();
assertEquals(Integer.valueOf(keyCount), key);
Iterator<Object> valuesIter = kvsReader.getCurrentValues().iterator();
int valCount = 0;
while (valuesIter.hasNext()) {
valCount++;
Integer val = (Integer) valuesIter.next();
assertEquals(Integer.valueOf(keyCount), val);
}
if (keyCount == 1) {
assertEquals(2, valCount);
} else if (keyCount == 2) {
assertEquals(4, valCount);
} else if (keyCount == 3) {
assertEquals(2, valCount);
} else if (keyCount == 4) {
fail("Key 4 should have been skipped");
} else if (keyCount == 5 || keyCount == 6 || keyCount == 7) {
assertEquals(2, valCount);
} else {
fail("Unexpected key");
}
}
getNextFromFinishedReader(kvsReader);
}
Aggregations