Examples with IOManagerAsync org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync used on opensource projects

Example 11 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class NonReusingHashJoinIteratorITCase method beforeTest.

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
    this.recordSerializer = TestData.getIntStringTupleSerializer();
    this.record1Comparator = TestData.getIntStringTupleComparator();
    this.record2Comparator = TestData.getIntStringTupleComparator();
    this.recordPairComparator = new GenericPairComparator(record1Comparator, record2Comparator);
    this.pairSerializer = new IntPairSerializer();
    this.pairComparator = new TestData.IntPairComparator();
    this.pairRecordPairComparator = new IntPairTuplePairComparator();
    this.recordPairPairComparator = new TupleIntPairPairComparator();
    this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
    this.ioManager = new IOManagerAsync();
}

Example 12 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class ReOpenableHashTableITCase method beforeTest.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Before
public void beforeTest() {
    this.recordBuildSideAccesssor = TestData.getIntIntTupleSerializer();
    this.recordProbeSideAccesssor = TestData.getIntIntTupleSerializer();
    this.recordBuildSideComparator = TestData.getIntIntTupleComparator();
    this.recordProbeSideComparator = TestData.getIntIntTupleComparator();
    this.pactRecordComparator = new GenericPairComparator(this.recordBuildSideComparator, this.recordProbeSideComparator);
    this.memoryManager = new MemoryManager(MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
    this.ioManager = new IOManagerAsync();
}

Example 13 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableTest method testBufferMissingForProbing.

// ------------------------------------------------------------------------
//  Tests
// ------------------------------------------------------------------------
/**
	 * This tests a combination of values that lead to a corner case situation where memory
	 * was missing and the computation deadlocked.
	 */
@Test
public void testBufferMissingForProbing() {
    final IOManager ioMan = new IOManagerAsync();
    try {
        final int pageSize = 32 * 1024;
        final int numSegments = 34;
        final int numRecords = 3400;
        final int recordLen = 270;
        final byte[] payload = new byte[recordLen - 8 - 4];
        List<MemorySegment> memory = getMemory(numSegments, pageSize);
        MutableHashTable<Tuple2<Long, byte[]>, Long> table = new MutableHashTable<>(buildSerializer, probeSerializer, buildComparator, probeComparator, pairComparator, memory, ioMan, 16, false);
        table.open(new TupleBytesIterator(payload, numRecords), new LongIterator(10000));
        try {
            while (table.nextRecord()) {
                MutableObjectIterator<Tuple2<Long, byte[]>> matches = table.getBuildSideIterator();
                while (matches.next() != null) ;
            }
        } catch (RuntimeException e) {
            if (!e.getMessage().contains("exceeded maximum number of recursions")) {
                e.printStackTrace();
                fail("Test failed with unexpected exception");
            }
        } finally {
            table.close();
        }
        checkNoTempFilesRemain(ioMan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 14 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableTest method testSpillingFreesOnlyOverflowSegments.

/**
	 * This tests the case where no additional partition buffers are used at the point when spilling
	 * is triggered, testing that overflow bucket buffers are taken into account when deciding which
	 * partition to spill.
	 */
@Test
public void testSpillingFreesOnlyOverflowSegments() {
    final IOManager ioMan = new IOManagerAsync();
    final TypeSerializer<ByteValue> serializer = ByteValueSerializer.INSTANCE;
    final TypeComparator<ByteValue> buildComparator = new ValueComparator<>(true, ByteValue.class);
    final TypeComparator<ByteValue> probeComparator = new ValueComparator<>(true, ByteValue.class);
    @SuppressWarnings("unchecked") final TypePairComparator<ByteValue, ByteValue> pairComparator = Mockito.mock(TypePairComparator.class);
    try {
        final int pageSize = 32 * 1024;
        final int numSegments = 34;
        List<MemorySegment> memory = getMemory(numSegments, pageSize);
        MutableHashTable<ByteValue, ByteValue> table = new MutableHashTable<>(serializer, serializer, buildComparator, probeComparator, pairComparator, memory, ioMan, 1, false);
        table.open(new ByteValueIterator(100000000), new ByteValueIterator(1));
        table.close();
        checkNoTempFilesRemain(ioMan);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 15 with IOManagerAsync

use of org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync in project flink by apache.

the class HashTableITCase method testSpillingHashJoinWithMassiveCollisionsIntPair.

@Test
public void testSpillingHashJoinWithMassiveCollisionsIntPair() throws IOException {
    // the following two values are known to have a hash-code collision on the initial level.
    // we use them to make sure one partition grows over-proportionally large
    final int REPEATED_VALUE_1 = 40559;
    final int REPEATED_VALUE_2 = 92882;
    final int REPEATED_VALUE_COUNT_BUILD = 200000;
    final int REPEATED_VALUE_COUNT_PROBE = 5;
    final int NUM_KEYS = 1000000;
    final int BUILD_VALS_PER_KEY = 3;
    final int PROBE_VALS_PER_KEY = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus 400k pairs with two colliding keys
    MutableObjectIterator<IntPair> build1 = new UniformIntPairGenerator(NUM_KEYS, BUILD_VALS_PER_KEY, false);
    MutableObjectIterator<IntPair> build2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, REPEATED_VALUE_COUNT_BUILD);
    MutableObjectIterator<IntPair> build3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, REPEATED_VALUE_COUNT_BUILD);
    List<MutableObjectIterator<IntPair>> builds = new ArrayList<MutableObjectIterator<IntPair>>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<IntPair> buildInput = new UnionIterator<IntPair>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<IntPair> probe1 = new UniformIntPairGenerator(NUM_KEYS, PROBE_VALS_PER_KEY, true);
    MutableObjectIterator<IntPair> probe2 = new ConstantsIntPairsIterator(REPEATED_VALUE_1, 17, 5);
    MutableObjectIterator<IntPair> probe3 = new ConstantsIntPairsIterator(REPEATED_VALUE_2, 23, 5);
    List<MutableObjectIterator<IntPair>> probes = new ArrayList<MutableObjectIterator<IntPair>>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<IntPair> probeInput = new UnionIterator<IntPair>(probes);
    // allocate the memory for the HashTable
    List<MemorySegment> memSegments;
    try {
        memSegments = this.memManager.allocatePages(MEM_OWNER, 896);
    } catch (MemoryAllocationException maex) {
        fail("Memory for the Join could not be provided.");
        return;
    }
    // create the I/O access for spilling
    IOManager ioManager = new IOManagerAsync();
    // create the map for validating the results
    HashMap<Integer, Long> map = new HashMap<Integer, Long>(NUM_KEYS);
    // ----------------------------------------------------------------------------------------
    final MutableHashTable<IntPair, IntPair> join = new MutableHashTable<IntPair, IntPair>(this.pairBuildSideAccesssor, this.pairProbeSideAccesssor, this.pairBuildSideComparator, this.pairProbeSideComparator, this.pairComparator, memSegments, ioManager);
    join.open(buildInput, probeInput);
    IntPair record;
    final IntPair recordReuse = new IntPair();
    while (join.nextRecord()) {
        int numBuildValues = 0;
        final IntPair probeRec = join.getCurrentProbeRecord();
        int key = probeRec.getKey();
        MutableObjectIterator<IntPair> buildSide = join.getBuildSideIterator();
        if ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues = 1;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getKey());
        } else {
            fail("No build side values found for a probe key.");
        }
        while ((record = buildSide.next(recordReuse)) != null) {
            numBuildValues++;
            Assert.assertEquals("Probe-side key was different than build-side key.", key, record.getKey());
        }
        Long contained = map.get(key);
        if (contained == null) {
            contained = Long.valueOf(numBuildValues);
        } else {
            contained = Long.valueOf(contained.longValue() + numBuildValues);
        }
        map.put(key, contained);
    }
    join.close();
    Assert.assertEquals("Wrong number of keys", NUM_KEYS, map.size());
    for (Map.Entry<Integer, Long> entry : map.entrySet()) {
        long val = entry.getValue();
        int key = entry.getKey();
        Assert.assertEquals("Wrong number of values in per-key cross product for key " + key, (key == REPEATED_VALUE_1 || key == REPEATED_VALUE_2) ? (PROBE_VALS_PER_KEY + REPEATED_VALUE_COUNT_PROBE) * (BUILD_VALS_PER_KEY + REPEATED_VALUE_COUNT_BUILD) : PROBE_VALS_PER_KEY * BUILD_VALS_PER_KEY, val);
    }
    // ----------------------------------------------------------------------------------------
    this.memManager.release(join.getFreedMemory());
}