Examples with MemoryManager org.apache.flink.runtime.memory.MemoryManager used on opensource projects

Example 56 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class GroupReduceCombineDriver method prepare.

@Override
public void prepare() throws Exception {
    final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
    if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE) {
        throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
    }
    final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
    this.serializer = serializerFactory.getSerializer();
    final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
    this.groupingComparator = this.taskContext.getDriverComparator(1);
    this.combiner = this.taskContext.getStub();
    this.output = this.taskContext.getOutputCollector();
    MemoryManager memManager = this.taskContext.getMemoryManager();
    final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
    this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);
    // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
    if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
        this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
    } else {
        this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
    }
}

Example 57 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class MetricUtilsTest method testManagedMemoryMetricsInitialization.

@Test
public void testManagedMemoryMetricsInitialization() throws MemoryAllocationException {
    final int maxMemorySize = 16284;
    final int numberOfAllocatedPages = 2;
    final int pageSize = 4096;
    final Object owner = new Object();
    final MemoryManager memoryManager = MemoryManager.create(maxMemorySize, pageSize);
    memoryManager.allocatePages(owner, numberOfAllocatedPages);
    final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(new TestingTaskSlotTable.TestingTaskSlotTableBuilder<Task>().memoryManagerGetterReturns(memoryManager).allActiveSlotAllocationIds(() -> Sets.newHashSet(new AllocationID())).build()).setManagedMemorySize(maxMemorySize).build();
    List<String> actualSubGroupPath = new ArrayList<>();
    final InterceptingOperatorMetricGroup metricGroup = new InterceptingOperatorMetricGroup() {

        @Override
        public MetricGroup addGroup(String name) {
            actualSubGroupPath.add(name);
            return this;
        }
    };
    MetricUtils.instantiateFlinkMemoryMetricGroup(metricGroup, taskManagerServices.getTaskSlotTable(), taskManagerServices::getManagedMemorySize);
    Gauge<Number> usedMetric = (Gauge<Number>) metricGroup.get("Used");
    Gauge<Number> maxMetric = (Gauge<Number>) metricGroup.get("Total");
    assertThat(usedMetric.getValue().intValue(), is(numberOfAllocatedPages * pageSize));
    assertThat(maxMetric.getValue().intValue(), is(maxMemorySize));
    assertThat(actualSubGroupPath, is(Arrays.asList(METRIC_GROUP_FLINK, METRIC_GROUP_MEMORY, METRIC_GROUP_MANAGED_MEMORY)));
}

Example 58 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class MultiInputSortingDataInput method wrapInputs.

public static <K> SelectableSortingInputs wrapInputs(TaskInvokable containingTask, StreamTaskInput<Object>[] sortingInputs, KeySelector<Object, K>[] keySelectors, TypeSerializer<Object>[] inputSerializers, TypeSerializer<K> keySerializer, StreamTaskInput<Object>[] passThroughInputs, MemoryManager memoryManager, IOManager ioManager, boolean objectReuse, double managedMemoryFraction, Configuration jobConfiguration, ExecutionConfig executionConfig) {
    int keyLength = keySerializer.getLength();
    final TypeComparator<Tuple2<byte[], StreamRecord<Object>>> comparator;
    DataOutputSerializer dataOutputSerializer;
    if (keyLength > 0) {
        dataOutputSerializer = new DataOutputSerializer(keyLength);
        comparator = new FixedLengthByteKeyComparator<>(keyLength);
    } else {
        dataOutputSerializer = new DataOutputSerializer(64);
        comparator = new VariableLengthByteKeyComparator<>();
    }
    List<Integer> passThroughInputIndices = Arrays.stream(passThroughInputs).map(StreamTaskInput::getInputIndex).collect(Collectors.toList());
    int numberOfInputs = sortingInputs.length + passThroughInputs.length;
    CommonContext commonContext = new CommonContext(sortingInputs);
    InputSelector inputSelector = new InputSelector(commonContext, numberOfInputs, passThroughInputIndices);
    StreamTaskInput<?>[] wrappedSortingInputs = IntStream.range(0, sortingInputs.length).mapToObj(idx -> {
        try {
            KeyAndValueSerializer<Object> keyAndValueSerializer = new KeyAndValueSerializer<>(inputSerializers[idx], keyLength);
            return new MultiInputSortingDataInput<>(commonContext, sortingInputs[idx], sortingInputs[idx].getInputIndex(), ExternalSorter.newBuilder(memoryManager, containingTask, keyAndValueSerializer, comparator, executionConfig).memoryFraction(managedMemoryFraction / numberOfInputs).enableSpilling(ioManager, jobConfiguration.get(AlgorithmOptions.SORT_SPILLING_THRESHOLD)).maxNumFileHandles(jobConfiguration.get(AlgorithmOptions.SPILLING_MAX_FAN) / numberOfInputs).objectReuse(objectReuse).largeRecords(true).build(), keySelectors[idx], keySerializer, dataOutputSerializer);
        } catch (MemoryAllocationException e) {
            throw new RuntimeException();
        }
    }).toArray(StreamTaskInput[]::new);
    StreamTaskInput<?>[] wrappedPassThroughInputs = Arrays.stream(passThroughInputs).map(input -> new ObservableStreamTaskInput<>(input, inputSelector)).toArray(StreamTaskInput[]::new);
    return new SelectableSortingInputs(wrappedSortingInputs, wrappedPassThroughInputs, inputSelector);
}

Example 59 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BinaryHashTableTest method testSparseProbeSpillingWithOuterJoin.

/*
     * Same test as {@link #testSparseProbeSpilling} but using a build-side outer join
     * that requires spilled build-side records to be returned and counted.
     */
@Test
public void testSparseProbeSpillingWithOuterJoin() throws IOException {
    final int numBuildKeys = 1000000;
    final int numBuildVals = 1;
    final int numProbeKeys = 20;
    final int numProbeVals = 1;
    MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numBuildKeys, numBuildVals, false);
    MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(96 * PAGE_SIZE).build();
    final BinaryHashTable table = new BinaryHashTable(conf, new Object(), this.buildSideSerializer, this.probeSideSerializer, new MyProjection(), new MyProjection(), memManager, 96 * PAGE_SIZE, ioManager, 24, 200000, true, HashJoinType.BUILD_OUTER, null, true, new boolean[] { true }, false);
    int expectedNumResults = (Math.max(numProbeKeys, numBuildKeys) * numBuildVals) * numProbeVals;
    int numRecordsInJoinResult = join(table, buildInput, new UniformBinaryRowGenerator(numProbeKeys, numProbeVals, true), true);
    Assert.assertEquals("Wrong number of records in join result.", expectedNumResults, numRecordsInJoinResult);
    table.close();
    table.free();
}

Example 60 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BinaryHashTableTest method testFailingHashJoinTooManyRecursions.

/*
     * This test is basically identical to the "testSpillingHashJoinWithMassiveCollisions" test, only that the number
     * of repeated values (causing bucket collisions) are large enough to make sure that their target partition no longer
     * fits into memory by itself and needs to be repartitioned in the recursion again.
     */
@Test
public void testFailingHashJoinTooManyRecursions() throws IOException {
    // the following two values are known to have a hash-code collision on the first recursion
    // level.
    // we use them to make sure one partition grows over-proportionally large
    final int repeatedValue1 = 40559;
    final int repeatedValue2 = 92882;
    final int repeatedValueCount = 3000000;
    final int numKeys = 1000000;
    final int buildValsPerKey = 3;
    final int probeValsPerKey = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key, plus
    // 400k pairs with two colliding keys
    MutableObjectIterator<BinaryRowData> build1 = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);
    MutableObjectIterator<BinaryRowData> build2 = new ConstantsKeyValuePairsIterator(repeatedValue1, 17, repeatedValueCount);
    MutableObjectIterator<BinaryRowData> build3 = new ConstantsKeyValuePairsIterator(repeatedValue2, 23, repeatedValueCount);
    List<MutableObjectIterator<BinaryRowData>> builds = new ArrayList<>();
    builds.add(build1);
    builds.add(build2);
    builds.add(build3);
    MutableObjectIterator<BinaryRowData> buildInput = new UnionIterator<>(builds);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<BinaryRowData> probe1 = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);
    MutableObjectIterator<BinaryRowData> probe2 = new ConstantsKeyValuePairsIterator(repeatedValue1, 17, repeatedValueCount);
    MutableObjectIterator<BinaryRowData> probe3 = new ConstantsKeyValuePairsIterator(repeatedValue2, 23, repeatedValueCount);
    List<MutableObjectIterator<BinaryRowData>> probes = new ArrayList<>();
    probes.add(probe1);
    probes.add(probe2);
    probes.add(probe3);
    MutableObjectIterator<BinaryRowData> probeInput = new UnionIterator<>(probes);
    // ----------------------------------------------------------------------------------------
    MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(896 * PAGE_SIZE).build();
    final BinaryHashTable table = newBinaryHashTable(this.buildSideSerializer, this.probeSideSerializer, new MyProjection(), new MyProjection(), memManager, 896 * PAGE_SIZE, ioManager);
    try {
        join(table, buildInput, probeInput);
        fail("Hash Join must have failed due to too many recursions.");
    } catch (Exception ex) {
    // expected
    }
    table.close();
    // ----------------------------------------------------------------------------------------
    table.free();
}