Examples with Record org.apache.flink.types.Record used on opensource projects

Example 1 with Record

use of org.apache.flink.types.Record in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method setup.

@Before
public void setup() {
    final ArrayList<IntStringPair> source = new ArrayList<IntStringPair>();
    // add elements to the source
    source.add(new IntStringPair(new IntValue(1), new StringValue("A")));
    source.add(new IntStringPair(new IntValue(2), new StringValue("B")));
    source.add(new IntStringPair(new IntValue(3), new StringValue("C")));
    source.add(new IntStringPair(new IntValue(3), new StringValue("D")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("E")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("F")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("G")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("H")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("I")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("J")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("K")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("L")));
    this.sourceIter = new MutableObjectIterator<Record>() {

        final Iterator<IntStringPair> it = source.iterator();

        @Override
        public Record next(Record reuse) throws IOException {
            if (it.hasNext()) {
                IntStringPair pair = it.next();
                reuse.setField(0, pair.getInteger());
                reuse.setField(1, pair.getString());
                return reuse;
            } else {
                return null;
            }
        }

        @Override
        public Record next() throws IOException {
            if (it.hasNext()) {
                IntStringPair pair = it.next();
                Record result = new Record(2);
                result.setField(0, pair.getInteger());
                result.setField(1, pair.getString());
                return result;
            } else {
                return null;
            }
        }
    };
    @SuppressWarnings("unchecked") final RecordComparator comparator = new RecordComparator(new int[] { 0 }, new Class[] { IntValue.class });
    this.psi = new NonReusingKeyGroupedIterator<Record>(this.sourceIter, comparator);
}

Example 2 with Record

use of org.apache.flink.types.Record in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method testFullIterationThroughAllValues.

@Test
public void testFullIterationThroughAllValues() throws IOException {
    try {
        // Key 1, Value A
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 2, Value B
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 3, Values C, D
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 4, Values E, F, G
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("F"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("G"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 5, Values H, I, J, K, L
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("J"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("K"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("L"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}

Example 3 with Record

use of org.apache.flink.types.Record in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method testMixedProgress.

@Test
public void testMixedProgress() throws Exception {
    try {
        // Progression only via nextKey() and hasNext() - Key 1, Value A
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        // Progression only through nextKey() - Key 2, Value B
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        // Progression first though haNext() and next(), then through hasNext() - Key 3, Values C, D
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Progression first via next() only, then hasNext() only Key 4, Values E, F, G
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        // Key 5, Values H, I, J, K, L
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        // end
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}

Example 4 with Record

use of org.apache.flink.types.Record in project flink by apache.

the class ReusingKeyGroupedIteratorTest method testHasNextDoesNotOverweiteCurrentRecord.

@Test
public void testHasNextDoesNotOverweiteCurrentRecord() throws Exception {
    try {
        Iterator<Record> valsIter = null;
        Record rec = null;
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        valsIter = this.psi.getValues();
        Assert.assertNotNull("Returned Iterator must not be null", valsIter);
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        valsIter = this.psi.getValues();
        Assert.assertNotNull("Returned Iterator must not be null", valsIter);
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        valsIter = this.psi.getValues();
        Assert.assertNotNull("Returned Iterator must not be null", valsIter);
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}

Example 5 with Record

use of org.apache.flink.types.Record in project flink by apache.

the class CrossTaskTest method testBlock2CrossTask.

@Test
public void testBlock2CrossTask() {
    int keyCnt1 = 10;
    int valCnt1 = 1;
    int keyCnt2 = 100;
    int valCnt2 = 4;
    final int expCnt = keyCnt1 * valCnt1 * keyCnt2 * valCnt2;
    setOutput(this.output);
    addInput(new UniformRecordGenerator(keyCnt1, valCnt1, false));
    addInput(new UniformRecordGenerator(keyCnt2, valCnt2, false));
    getTaskConfig().setDriverStrategy(DriverStrategy.NESTEDLOOP_BLOCKED_OUTER_SECOND);
    getTaskConfig().setRelativeMemoryDriver(cross_frac);
    final CrossDriver<Record, Record, Record> testTask = new CrossDriver<>();
    try {
        testDriver(testTask, MockCrossStub.class);
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Test failed due to an exception.");
    }
    Assert.assertEquals("Wrong result size.", expCnt, this.output.getNumberOfRecords());
}