Examples with DummyInvokable org.apache.flink.runtime.operators.testutils.DummyInvokable used on opensource projects

Example 36 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerTest method allocateAllSingle.

@Test
public void allocateAllSingle() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        List<MemorySegment> segments = new ArrayList<MemorySegment>();
        try {
            for (int i = 0; i < NUM_PAGES; i++) {
                segments.add(this.memoryManager.allocatePages(mockInvoke, 1).get(0));
            }
        } catch (MemoryAllocationException e) {
            fail("Unable to allocate memory");
        }
        this.memoryManager.release(segments);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 37 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerTest method allocateTooMuch.

@Test
public void allocateTooMuch() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        List<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);
        try {
            this.memoryManager.allocatePages(mockInvoke, 1);
            Assert.fail("Expected MemoryAllocationException.");
        } catch (MemoryAllocationException maex) {
        // expected
        }
        Assert.assertTrue("The previously allocated segments were not valid any more.", allMemorySegmentsValid(segs));
        this.memoryManager.releaseAll(mockInvoke);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 38 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class LargeRecordHandlerTest method testRecordHandlerSingleKey.

@Test
public void testRecordHandlerSingleKey() {
    final IOManager ioMan = new IOManagerAsync();
    final int PAGE_SIZE = 4 * 1024;
    final int NUM_PAGES = 24;
    final int NUM_RECORDS = 25000;
    try {
        final MemoryManager memMan = new MemoryManager(NUM_PAGES * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        final AbstractInvokable owner = new DummyInvokable();
        final List<MemorySegment> initialMemory = memMan.allocatePages(owner, 6);
        final List<MemorySegment> sortMemory = memMan.allocatePages(owner, NUM_PAGES - 6);
        final TupleTypeInfo<Tuple2<Long, String>> typeInfo = (TupleTypeInfo<Tuple2<Long, String>>) TypeInfoParser.<Tuple2<Long, String>>parse("Tuple2<Long, String>");
        final TypeSerializer<Tuple2<Long, String>> serializer = typeInfo.createSerializer(new ExecutionConfig());
        final TypeComparator<Tuple2<Long, String>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        LargeRecordHandler<Tuple2<Long, String>> handler = new LargeRecordHandler<Tuple2<Long, String>>(serializer, comparator, ioMan, memMan, initialMemory, owner, 128);
        assertFalse(handler.hasData());
        // add the test data
        Random rnd = new Random();
        for (int i = 0; i < NUM_RECORDS; i++) {
            long val = rnd.nextLong();
            handler.addRecord(new Tuple2<Long, String>(val, String.valueOf(val)));
            assertTrue(handler.hasData());
        }
        MutableObjectIterator<Tuple2<Long, String>> sorted = handler.finishWriteAndSortKeys(sortMemory);
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        Tuple2<Long, String> previous = null;
        Tuple2<Long, String> next;
        while ((next = sorted.next(null)) != null) {
            // key and value must be equal
            assertTrue(next.f0.equals(Long.parseLong(next.f1)));
            // order must be correct
            if (previous != null) {
                assertTrue(previous.f0 <= next.f0);
            }
            previous = next;
        }
        handler.close();
        assertFalse(handler.hasData());
        handler.close();
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        assertTrue(memMan.verifyEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 39 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class LargeRecordHandlerTest method testEmptyRecordHandler.

@Test
public void testEmptyRecordHandler() {
    final IOManager ioMan = new IOManagerAsync();
    final int PAGE_SIZE = 4 * 1024;
    final int NUM_PAGES = 50;
    try {
        final MemoryManager memMan = new MemoryManager(NUM_PAGES * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        final AbstractInvokable owner = new DummyInvokable();
        final List<MemorySegment> memory = memMan.allocatePages(owner, NUM_PAGES);
        final TupleTypeInfo<Tuple2<Long, String>> typeInfo = (TupleTypeInfo<Tuple2<Long, String>>) TypeInfoParser.<Tuple2<Long, String>>parse("Tuple2<Long, String>");
        final TypeSerializer<Tuple2<Long, String>> serializer = typeInfo.createSerializer(new ExecutionConfig());
        final TypeComparator<Tuple2<Long, String>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        LargeRecordHandler<Tuple2<Long, String>> handler = new LargeRecordHandler<Tuple2<Long, String>>(serializer, comparator, ioMan, memMan, memory, owner, 128);
        assertFalse(handler.hasData());
        handler.close();
        assertFalse(handler.hasData());
        handler.close();
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        assertTrue(memMan.verifyEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 40 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class NormalizedKeySorterTest method testSortShortStringKeys.

@Test
public void testSortShortStringKeys() throws Exception {
    final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
    final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
    @SuppressWarnings("unchecked") TypeComparator<Tuple2<Integer, String>> accessors = TestData.getIntStringTupleTypeInfo().createComparator(new int[] { 1 }, new boolean[] { true }, 0, null);
    NormalizedKeySorter<Tuple2<Integer, String>> sorter = new NormalizedKeySorter<>(TestData.getIntStringTupleSerializer(), accessors, memory);
    TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, 5, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
    // write the records
    Tuple2<Integer, String> record = new Tuple2<>();
    do {
        generator.next(record);
    } while (sorter.write(record));
    QuickSort qs = new QuickSort();
    qs.sort(sorter);
    MutableObjectIterator<Tuple2<Integer, String>> iter = sorter.getIterator();
    Tuple2<Integer, String> readTarget = new Tuple2<>();
    iter.next(readTarget);
    String last = readTarget.f1;
    while ((readTarget = iter.next(readTarget)) != null) {
        String current = readTarget.f1;
        final int cmp = last.compareTo(current);
        if (cmp > 0) {
            Assert.fail("Next value is not larger or equal to previous value.");
        }
        last = current;
    }
    // release the memory occupied by the buffers
    sorter.dispose();
    this.memoryManager.release(memory);
}