Examples with IOManagerAsync org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync used on opensource projects

Example 26 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class ChannelViewsTest method beforeTest.

// --------------------------------------------------------------------------------------------
@Before
public void beforeTest() {
    this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, MEMORY_PAGE_SIZE, MemoryType.HEAP, true);
    this.ioManager = new IOManagerAsync();
}

Example 27 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class FileChannelStreamsTest method testCloseAndDeleteOutputView.

@Test
public void testCloseAndDeleteOutputView() {
    final IOManager ioManager = new IOManagerAsync();
    try {
        MemoryManager memMan = new MemoryManager(4 * 16 * 1024, 1, 16 * 1024, MemoryType.HEAP, true);
        List<MemorySegment> memory = new ArrayList<MemorySegment>();
        memMan.allocatePages(new DummyInvokable(), memory, 4);
        FileIOChannel.ID channel = ioManager.createChannel();
        BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
        FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
        new StringValue("Some test text").write(out);
        // close for the first time, make sure all memory returns
        out.close();
        assertTrue(memMan.verifyEmpty());
        // close again, should not cause an exception
        out.close();
        // delete, make sure file is removed
        out.closeAndDelete();
        assertFalse(new File(channel.getPath()).exists());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioManager.shutdown();
    }
}

Example 28 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableITCase method testInMemoryMutableHashTableIntPair.

// ============================================================================================
//                                 Integer Pairs based Tests
// ============================================================================================
@Test
public void testInMemoryMutableHashTableIntPair() throws IOException {
    final int NUM_KEYS = 100000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<IntPair> buildInput = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probeInput = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // create the I/O access for spilling
    final IOManager ioManager = new IOManagerAsync();
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    final IntPair recordReuse = new IntPair();
    int numRecordsInJoinResult = 0;
    while (join.nextRecord()) {
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        while (buildSide.next(recordReuse) != null) {
            numRecordsInJoinResult++;
        }
    }
    Assert.assertEquals("Wrong number of records in join result.", NUM_KEYS * BUILD_VALS_PER_KEY * PROBE_VALS_PER_KEY, numRecordsInJoinResult);
    join.close();
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}

Example 29 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableTest method testSpillingWhenBuildingTableWithoutOverflow.

/**
	 * Tests that the MutableHashTable spills its partitions when creating the initial table
	 * without overflow segments in the partitions. This means that the records are large.
	 */
@Test
public void testSpillingWhenBuildingTableWithoutOverflow() throws Exception {
    final IOManager ioMan = new IOManagerAsync();
    final TypeSerializer<byte[]> serializer = BytePrimitiveArraySerializer.INSTANCE;
    final TypeComparator<byte[]> buildComparator = new BytePrimitiveArrayComparator(true);
    final TypeComparator<byte[]> probeComparator = new BytePrimitiveArrayComparator(true);
    @SuppressWarnings("unchecked") final TypePairComparator<byte[], byte[]> pairComparator = new GenericPairComparator<>(new BytePrimitiveArrayComparator(true), new BytePrimitiveArrayComparator(true));
    final int pageSize = 128;
    final int numSegments = 33;
    List<MemorySegment> memory = getMemory(numSegments, pageSize);
    MutableHashTable<byte[], byte[]> table = new MutableHashTable<byte[], byte[]>(serializer, serializer, buildComparator, probeComparator, pairComparator, memory, ioMan, 1, false);
    int numElements = 9;
    table.open(new CombiningIterator<byte[]>(new ByteArrayIterator(numElements, 128, (byte) 0), new ByteArrayIterator(numElements, 128, (byte) 1)), new CombiningIterator<byte[]>(new ByteArrayIterator(1, 128, (byte) 0), new ByteArrayIterator(1, 128, (byte) 1)));
    while (table.nextRecord()) {
        MutableObjectIterator<byte[]> iterator = table.getBuildSideIterator();
        int counter = 0;
        while (iterator.next() != null) {
            counter++;
        }
        // check that we retrieve all our elements
        Assert.assertEquals(numElements, counter);
    }
    table.close();
}

Example 30 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableITCase method setup.

@Before
public void setup() {
    final int[] keyPos = new int[] { 0 };
    @SuppressWarnings("unchecked") final Class<? extends Value>[] keyType = (Class<? extends Value>[]) new Class[] { IntValue.class };
    this.recordBuildSideAccesssor = RecordSerializer.get();
    this.recordProbeSideAccesssor = RecordSerializer.get();
    this.recordBuildSideComparator = new RecordComparator(keyPos, keyType);
    this.recordProbeSideComparator = new RecordComparator(keyPos, keyType);
    this.pactRecordComparator = new RecordPairComparatorFirstInt();
    this.pairBuildSideAccesssor = new IntPairSerializer();
    this.pairProbeSideAccesssor = new IntPairSerializer();
    this.pairBuildSideComparator = new IntPairComparator();
    this.pairProbeSideComparator = new IntPairComparator();
    this.pairComparator = new IntPairPairComparator();
    this.memManager = new MemoryManager(32 * 1024 * 1024, 1);
    this.ioManager = new IOManagerAsync();
}