Examples with IOManagerAsync org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync used on opensource projects

Example 36 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class LargeRecordHandlerTest method testRecordHandlerSingleKey.

@Test
public void testRecordHandlerSingleKey() {
    final IOManager ioMan = new IOManagerAsync();
    final int PAGE_SIZE = 4 * 1024;
    final int NUM_PAGES = 24;
    final int NUM_RECORDS = 25000;
    try {
        final MemoryManager memMan = new MemoryManager(NUM_PAGES * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        final AbstractInvokable owner = new DummyInvokable();
        final List<MemorySegment> initialMemory = memMan.allocatePages(owner, 6);
        final List<MemorySegment> sortMemory = memMan.allocatePages(owner, NUM_PAGES - 6);
        final TupleTypeInfo<Tuple2<Long, String>> typeInfo = (TupleTypeInfo<Tuple2<Long, String>>) TypeInfoParser.<Tuple2<Long, String>>parse("Tuple2<Long, String>");
        final TypeSerializer<Tuple2<Long, String>> serializer = typeInfo.createSerializer(new ExecutionConfig());
        final TypeComparator<Tuple2<Long, String>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        LargeRecordHandler<Tuple2<Long, String>> handler = new LargeRecordHandler<Tuple2<Long, String>>(serializer, comparator, ioMan, memMan, initialMemory, owner, 128);
        assertFalse(handler.hasData());
        // add the test data
        Random rnd = new Random();
        for (int i = 0; i < NUM_RECORDS; i++) {
            long val = rnd.nextLong();
            handler.addRecord(new Tuple2<Long, String>(val, String.valueOf(val)));
            assertTrue(handler.hasData());
        }
        MutableObjectIterator<Tuple2<Long, String>> sorted = handler.finishWriteAndSortKeys(sortMemory);
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        Tuple2<Long, String> previous = null;
        Tuple2<Long, String> next;
        while ((next = sorted.next(null)) != null) {
            // key and value must be equal
            assertTrue(next.f0.equals(Long.parseLong(next.f1)));
            // order must be correct
            if (previous != null) {
                assertTrue(previous.f0 <= next.f0);
            }
            previous = next;
        }
        handler.close();
        assertFalse(handler.hasData());
        handler.close();
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        assertTrue(memMan.verifyEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 37 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class LargeRecordHandlerTest method testEmptyRecordHandler.

@Test
public void testEmptyRecordHandler() {
    final IOManager ioMan = new IOManagerAsync();
    final int PAGE_SIZE = 4 * 1024;
    final int NUM_PAGES = 50;
    try {
        final MemoryManager memMan = new MemoryManager(NUM_PAGES * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        final AbstractInvokable owner = new DummyInvokable();
        final List<MemorySegment> memory = memMan.allocatePages(owner, NUM_PAGES);
        final TupleTypeInfo<Tuple2<Long, String>> typeInfo = (TupleTypeInfo<Tuple2<Long, String>>) TypeInfoParser.<Tuple2<Long, String>>parse("Tuple2<Long, String>");
        final TypeSerializer<Tuple2<Long, String>> serializer = typeInfo.createSerializer(new ExecutionConfig());
        final TypeComparator<Tuple2<Long, String>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        LargeRecordHandler<Tuple2<Long, String>> handler = new LargeRecordHandler<Tuple2<Long, String>>(serializer, comparator, ioMan, memMan, memory, owner, 128);
        assertFalse(handler.hasData());
        handler.close();
        assertFalse(handler.hasData());
        handler.close();
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        assertTrue(memMan.verifyEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 38 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class SpillingResettableIteratorTest method startup.

@Before
public void startup() {
    // set up IO and memory manager
    this.memman = new MemoryManager(MEMORY_CAPACITY, 1, 32 * 1024, MemoryType.HEAP, true);
    this.ioman = new IOManagerAsync();
    // create test objects
    ArrayList<IntValue> objects = new ArrayList<IntValue>(NUM_TESTRECORDS);
    for (int i = 0; i < NUM_TESTRECORDS; ++i) {
        IntValue tmp = new IntValue(i);
        objects.add(tmp);
    }
    this.reader = objects.iterator();
}

Example 39 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class SpillingResettableMutableObjectIteratorTest method startup.

@Before
public void startup() {
    // set up IO and memory manager
    this.memman = new MemoryManager(MEMORY_CAPACITY, 1);
    this.ioman = new IOManagerAsync();
    // create test objects
    final ArrayList<Record> objects = new ArrayList<Record>(NUM_TESTRECORDS);
    for (int i = 0; i < NUM_TESTRECORDS; ++i) {
        Record tmp = new Record(new IntValue(i));
        objects.add(tmp);
    }
    this.reader = new MutableObjectIteratorWrapper(objects.iterator());
}

Example 40 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class CombiningUnilateralSortMergerITCase method beforeTest.

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
    this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
    this.ioManager = new IOManagerAsync();
    this.serializerFactory1 = TestData.getIntStringTupleSerializerFactory();
    this.comparator1 = TestData.getIntStringTupleComparator();
    this.serializerFactory2 = TestData.getIntIntTupleSerializerFactory();
    this.comparator2 = TestData.getIntIntTupleComparator();
}