Search in sources :

Example 41 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class MassiveStringValueSorting method testStringValueTuplesSorting.

@SuppressWarnings("unchecked")
public void testStringValueTuplesSorting() {
    final int numStrings = 300000;
    File input = null;
    File sorted = null;
    try {
        // the source file
        input = generateFileWithStringTuples(numStrings, "http://some-uri.com/that/is/a/common/prefix/to/all");
        // the sorted file
        sorted = File.createTempFile("sorted_strings", "txt");
        String[] command = { "/bin/bash", "-c", "export LC_ALL=\"C\" && cat \"" + input.getAbsolutePath() + "\" | sort > \"" + sorted.getAbsolutePath() + "\"" };
        Process p = null;
        try {
            p = Runtime.getRuntime().exec(command);
            int retCode = p.waitFor();
            if (retCode != 0) {
                throw new Exception("Command failed with return code " + retCode);
            }
            p = null;
        } finally {
            if (p != null) {
                p.destroy();
            }
        }
        // sort the data
        Sorter<Tuple2<StringValue, StringValue[]>> sorter = null;
        BufferedReader reader = null;
        BufferedReader verifyReader = null;
        MemoryManager mm = null;
        try (IOManager ioMan = new IOManagerAsync()) {
            mm = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build();
            TupleTypeInfo<Tuple2<StringValue, StringValue[]>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, StringValue[]>>) new TypeHint<Tuple2<StringValue, StringValue[]>>() {
            }.getTypeInfo();
            TypeSerializer<Tuple2<StringValue, StringValue[]>> serializer = typeInfo.createSerializer(new ExecutionConfig());
            TypeComparator<Tuple2<StringValue, StringValue[]>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
            reader = new BufferedReader(new FileReader(input));
            MutableObjectIterator<Tuple2<StringValue, StringValue[]>> inputIterator = new StringValueTupleReaderMutableObjectIterator(reader);
            sorter = ExternalSorter.newBuilder(mm, new DummyInvokable(), serializer, comparator).maxNumFileHandles(4).enableSpilling(ioMan, 0.8f).memoryFraction(1.0).objectReuse(false).largeRecords(true).build(inputIterator);
            // use this part to verify that all if good when sorting in memory
            // List<MemorySegment> memory = mm.allocatePages(new DummyInvokable(),
            // mm.computeNumberOfPages(1024*1024*1024));
            // NormalizedKeySorter<Tuple2<String, String[]>> nks = new
            // NormalizedKeySorter<Tuple2<String,String[]>>(serializer, comparator, memory);
            // 
            // {
            // Tuple2<String, String[]> wi = new Tuple2<String, String[]>("", new
            // String[0]);
            // while ((wi = inputIterator.next(wi)) != null) {
            // Assert.assertTrue(nks.write(wi));
            // }
            // 
            // new QuickSort().sort(nks);
            // }
            // 
            // MutableObjectIterator<Tuple2<String, String[]>> sortedData =
            // nks.getIterator();
            MutableObjectIterator<Tuple2<StringValue, StringValue[]>> sortedData = sorter.getIterator();
            reader.close();
            // verify
            verifyReader = new BufferedReader(new FileReader(sorted));
            MutableObjectIterator<Tuple2<StringValue, StringValue[]>> verifyIterator = new StringValueTupleReaderMutableObjectIterator(verifyReader);
            Tuple2<StringValue, StringValue[]> nextVerify = new Tuple2<StringValue, StringValue[]>(new StringValue(), new StringValue[0]);
            Tuple2<StringValue, StringValue[]> nextFromFlinkSort = new Tuple2<StringValue, StringValue[]>(new StringValue(), new StringValue[0]);
            int num = 0;
            while ((nextVerify = verifyIterator.next(nextVerify)) != null) {
                num++;
                nextFromFlinkSort = sortedData.next(nextFromFlinkSort);
                Assert.assertNotNull(nextFromFlinkSort);
                Assert.assertEquals(nextVerify.f0, nextFromFlinkSort.f0);
                Assert.assertArrayEquals(nextVerify.f1, nextFromFlinkSort.f1);
            }
            Assert.assertNull(sortedData.next(nextFromFlinkSort));
            Assert.assertEquals(numStrings, num);
        } finally {
            if (reader != null) {
                reader.close();
            }
            if (verifyReader != null) {
                verifyReader.close();
            }
            if (sorter != null) {
                sorter.close();
            }
            if (mm != null) {
                mm.shutdown();
            }
        }
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        if (input != null) {
            // noinspection ResultOfMethodCallIgnored
            input.delete();
        }
        if (sorted != null) {
            // noinspection ResultOfMethodCallIgnored
            sorted.delete();
        }
    }
}
Also used : TypeHint(org.apache.flink.api.common.typeinfo.TypeHint) IOManager(org.apache.flink.runtime.io.disk.iomanager.IOManager) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) MemoryManager(org.apache.flink.runtime.memory.MemoryManager) TypeHint(org.apache.flink.api.common.typeinfo.TypeHint) IOException(java.io.IOException) TupleTypeInfo(org.apache.flink.api.java.typeutils.TupleTypeInfo) IOManagerAsync(org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync) Tuple2(org.apache.flink.api.java.tuple.Tuple2) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) DummyInvokable(org.apache.flink.runtime.operators.testutils.DummyInvokable) StringValue(org.apache.flink.types.StringValue) File(java.io.File)

Example 42 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method testFullIterationThroughAllValues.

@Test
public void testFullIterationThroughAllValues() throws IOException {
    try {
        // Key 1, Value A
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 2, Value B
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 3, Values C, D
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 4, Values E, F, G
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("F"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("G"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 5, Values H, I, J, K, L
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("J"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("K"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("L"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}
Also used : Record(org.apache.flink.types.Record) StringValue(org.apache.flink.types.StringValue) IntValue(org.apache.flink.types.IntValue) NoSuchElementException(java.util.NoSuchElementException) IOException(java.io.IOException) NoSuchElementException(java.util.NoSuchElementException) Test(org.junit.Test)

Example 43 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method testMixedProgress.

@Test
public void testMixedProgress() throws Exception {
    try {
        // Progression only via nextKey() and hasNext() - Key 1, Value A
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        // Progression only through nextKey() - Key 2, Value B
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        // Progression first though haNext() and next(), then through hasNext() - Key 3, Values
        // C, D
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Progression first via next() only, then hasNext() only Key 4, Values E, F, G
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        // Key 5, Values H, I, J, K, L
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        // end
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}
Also used : Record(org.apache.flink.types.Record) StringValue(org.apache.flink.types.StringValue) IntValue(org.apache.flink.types.IntValue) IOException(java.io.IOException) NoSuchElementException(java.util.NoSuchElementException) Test(org.junit.Test)

Example 44 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method setup.

@Before
public void setup() {
    final ArrayList<IntStringPair> source = new ArrayList<IntStringPair>();
    // add elements to the source
    source.add(new IntStringPair(new IntValue(1), new StringValue("A")));
    source.add(new IntStringPair(new IntValue(2), new StringValue("B")));
    source.add(new IntStringPair(new IntValue(3), new StringValue("C")));
    source.add(new IntStringPair(new IntValue(3), new StringValue("D")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("E")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("F")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("G")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("H")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("I")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("J")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("K")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("L")));
    this.sourceIter = new MutableObjectIterator<Record>() {

        final Iterator<IntStringPair> it = source.iterator();

        @Override
        public Record next(Record reuse) throws IOException {
            if (it.hasNext()) {
                IntStringPair pair = it.next();
                reuse.setField(0, pair.getInteger());
                reuse.setField(1, pair.getString());
                return reuse;
            } else {
                return null;
            }
        }

        @Override
        public Record next() throws IOException {
            if (it.hasNext()) {
                IntStringPair pair = it.next();
                Record result = new Record(2);
                result.setField(0, pair.getInteger());
                result.setField(1, pair.getString());
                return result;
            } else {
                return null;
            }
        }
    };
    @SuppressWarnings("unchecked") final RecordComparator comparator = new RecordComparator(new int[] { 0 }, new Class[] { IntValue.class });
    this.psi = new NonReusingKeyGroupedIterator<Record>(this.sourceIter, comparator);
}
Also used : ArrayList(java.util.ArrayList) IOException(java.io.IOException) RecordComparator(org.apache.flink.runtime.testutils.recordutils.RecordComparator) Record(org.apache.flink.types.Record) StringValue(org.apache.flink.types.StringValue) IntValue(org.apache.flink.types.IntValue) Before(org.junit.Before)

Example 45 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class ReusingKeyGroupedIteratorTest method testHasNextDoesNotOverweiteCurrentRecord.

@Test
public void testHasNextDoesNotOverweiteCurrentRecord() throws Exception {
    try {
        Iterator<Record> valsIter = null;
        Record rec = null;
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        valsIter = this.psi.getValues();
        Assert.assertNotNull("Returned Iterator must not be null", valsIter);
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        valsIter = this.psi.getValues();
        Assert.assertNotNull("Returned Iterator must not be null", valsIter);
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must not have another value.", valsIter.hasNext());
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        valsIter = this.psi.getValues();
        Assert.assertNotNull("Returned Iterator must not be null", valsIter);
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), rec.getField(1, StringValue.class));
        rec = valsIter.next();
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator's value iterator must have another value.", valsIter.hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, rec.getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), rec.getField(1, StringValue.class));
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}
Also used : Record(org.apache.flink.types.Record) StringValue(org.apache.flink.types.StringValue) IOException(java.io.IOException) TraversableOnceException(org.apache.flink.util.TraversableOnceException) NoSuchElementException(java.util.NoSuchElementException) Test(org.junit.Test)

Aggregations

StringValue (org.apache.flink.types.StringValue)88 Test (org.junit.Test)61 IntValue (org.apache.flink.types.IntValue)35 LongValue (org.apache.flink.types.LongValue)21 IOException (java.io.IOException)17 ArrayList (java.util.ArrayList)15 Record (org.apache.flink.types.Record)13 TupleTypeInfo (org.apache.flink.api.java.typeutils.TupleTypeInfo)12 ExecutionEnvironment (org.apache.flink.api.java.ExecutionEnvironment)11 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)11 DoubleValue (org.apache.flink.types.DoubleValue)11 Value (org.apache.flink.types.Value)10 Tuple3 (org.apache.flink.api.java.tuple.Tuple3)9 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)7 Plan (org.apache.flink.api.common.Plan)7 Configuration (org.apache.flink.configuration.Configuration)7 FileInputSplit (org.apache.flink.core.fs.FileInputSplit)7 NoSuchElementException (java.util.NoSuchElementException)6 File (java.io.File)5 JobExecutionResult (org.apache.flink.api.common.JobExecutionResult)5