Examples with MemoryManager org.apache.flink.runtime.memory.MemoryManager used on opensource projects

Example 31 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BinaryHashTableTest method testSpillingHashJoinOneRecursionPerformance.

@Test
public void testSpillingHashJoinOneRecursionPerformance() throws IOException {
    final int numKeys = 1000000;
    final int buildValsPerKey = 3;
    final int probeValsPerKey = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<BinaryRowData> probeInput = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);
    // allocate the memory for the HashTable
    MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(200 * PAGE_SIZE).setPageSize(PAGE_SIZE).build();
    final BinaryHashTable table = newBinaryHashTable(this.buildSideSerializer, this.probeSideSerializer, new MyProjection(), new MyProjection(), memManager, 100 * PAGE_SIZE, ioManager);
    // ----------------------------------------------------------------------------------------
    int numRecordsInJoinResult = join(table, buildInput, probeInput);
    Assert.assertEquals("Wrong number of records in join result.", numKeys * buildValsPerKey * probeValsPerKey, numRecordsInJoinResult);
    table.close();
    // ----------------------------------------------------------------------------------------
    table.free();
}

Example 32 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BinaryHashTableTest method testHashWithBuildSideOuterJoin2.

@Test
public void testHashWithBuildSideOuterJoin2() throws Exception {
    final int numKeys = 40000;
    final int buildValsPerKey = 2;
    final int probeValsPerKey = 1;
    // The keys of probe and build sides are overlapped, so there would be none unmatched build
    // elements
    // after probe phase, make sure build side outer join works well in this case.
    // create a build input that gives 80000 pairs with 2 values sharing the same key
    MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);
    // create a probe input that gives 40000 pairs with 1 values sharing a key
    MutableObjectIterator<BinaryRowData> probeInput = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);
    // allocate the memory for the HashTable
    MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(35 * PAGE_SIZE).build();
    final BinaryHashTable table = newBinaryHashTable(this.buildSideSerializer, this.probeSideSerializer, new MyProjection(), new MyProjection(), memManager, 33 * PAGE_SIZE, ioManager);
    // ----------------------------------------------------------------------------------------
    int numRecordsInJoinResult = join(table, buildInput, probeInput, true);
    Assert.assertEquals("Wrong number of records in join result.", numKeys * buildValsPerKey * probeValsPerKey, numRecordsInJoinResult);
    table.close();
    table.free();
}

Example 33 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class HashAggTest method before.

@Before
public void before() throws Exception {
    ioManager = new IOManagerAsync();
    operator = new SumHashAggTestOperator(40 * 32 * 1024) {

        @Override
        Object getOwner() {
            return HashAggTest.this;
        }

        @Override
        MemoryManager getMemoryManager() {
            return memoryManager;
        }

        @Override
        Collector<StreamRecord<RowData>> getOutput() {
            return new Collector<StreamRecord<RowData>>() {

                @Override
                public void collect(StreamRecord<RowData> record) {
                    RowData row = record.getValue();
                    outputMap.put(row.isNullAt(0) ? null : row.getInt(0), row.isNullAt(1) ? null : row.getLong(1));
                }

                @Override
                public void close() {
                }
            };
        }

        @Override
        Configuration getConf() {
            return new Configuration();
        }

        @Override
        public IOManager getIOManager() {
            return ioManager;
        }
    };
    operator.open();
}

Example 34 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BinaryHashTableTest method testSpillingHashJoinWithTwoRecursions.

/*
     * This test is basically identical to the "testSpillingHashJoinWithMassiveCollisions" test, only that the number
     * of repeated values (causing bucket collisions) are large enough to make sure that their target partition no longer
     * fits into memory by itself and needs to be repartitioned in the recursion again.
     */
@Test
public void testSpillingHashJoinWithTwoRecursions() throws IOException {
    // the following two values are known to have a hash-code collision on the first recursion
    // level.
    // we use them to make sure one partition grows over-proportionally large
    final int repeatedValue1 = 40559;
    final int repeatedValue2 = 92882;
    final int repeatedValueCountBuild = 200000;
    final int repeatedValueCountProbe = 5;
    final int numKeys = 1000000;
    final int buildValsPerKey = 3;
    final int probeValsPerKey = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus
    // 400k pairs with two colliding keys
    MutableObjectIterator<BinaryRowData> build1 = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);
    MutableObjectIterator<BinaryRowData> build2 = new ConstantsKeyValuePairsIterator(repeatedValue1, 17, repeatedValueCountBuild);
    MutableObjectIterator<BinaryRowData> build3 = new ConstantsKeyValuePairsIterator(repeatedValue2, 23, repeatedValueCountBuild);
    List<MutableObjectIterator<BinaryRowData>> builds = new ArrayList<>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<BinaryRowData> buildInput = new UnionIterator<>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<BinaryRowData> probe1 = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);
    MutableObjectIterator<BinaryRowData> probe2 = new ConstantsKeyValuePairsIterator(repeatedValue1, 17, 5);
    MutableObjectIterator<BinaryRowData> probe3 = new ConstantsKeyValuePairsIterator(repeatedValue2, 23, 5);
    List<MutableObjectIterator<BinaryRowData>> probes = new ArrayList<>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<BinaryRowData> probeInput = new UnionIterator<>(probes);
    // create the map for validating the results
    HashMap<Integer, Long> map = new HashMap<>(numKeys);
    // ----------------------------------------------------------------------------------------
    MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(896 * PAGE_SIZE).build();
    final BinaryHashTable table = newBinaryHashTable(this.buildSideSerializer, this.probeSideSerializer, new MyProjection(), new MyProjection(), memManager, 896 * PAGE_SIZE, ioManager);
    final BinaryRowData recordReuse = new BinaryRowData(2);
    BinaryRowData buildRow = buildSideSerializer.createInstance();
    while ((buildRow = buildInput.next(buildRow)) != null) {
        table.putBuildRow(buildRow);
    }
    table.endBuild();
    BinaryRowData probeRow = probeSideSerializer.createInstance();
    while ((probeRow = probeInput.next(probeRow)) != null) {
        if (table.tryProbe(probeRow)) {
            testJoin(table, map);
        }
    }
    while (table.nextMatching()) {
        testJoin(table, map);
    }
    table.close();
    Assert.assertEquals("Wrong number of keys", numKeys, map.size());
    for (Map.Entry<Integer, Long> entry : map.entrySet()) {
        long val = entry.getValue();
        int key = entry.getKey();
        Assert.assertEquals("Wrong number of values in per-key cross product for key " + key, (key == repeatedValue1 || key == repeatedValue2) ? (probeValsPerKey + repeatedValueCountProbe) * (buildValsPerKey + repeatedValueCountBuild) : probeValsPerKey * buildValsPerKey, val);
    }
    // ----------------------------------------------------------------------------------------
    table.free();
}

Example 35 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class TaskSlotTest method testTaskSlotClosedOnlyWhenAddedTasksTerminated.

@Test
public void testTaskSlotClosedOnlyWhenAddedTasksTerminated() throws Exception {
    try (TaskSlot<TaskSlotPayload> taskSlot = createTaskSlot()) {
        taskSlot.markActive();
        TestingTaskSlotPayload task = new TestingTaskSlotPayload(JOB_ID, new ExecutionAttemptID(), ALLOCATION_ID);
        taskSlot.add(task);
        CompletableFuture<Void> closingFuture = taskSlot.closeAsync();
        task.waitForFailure();
        MemoryManager memoryManager = taskSlot.getMemoryManager();
        assertThat(closingFuture.isDone(), is(false));
        assertThat(memoryManager.isShutdown(), is(false));
        task.terminate();
        closingFuture.get();
        assertThat(memoryManager.isShutdown(), is(true));
    }
}