Examples with MemoryManager org.apache.flink.runtime.memory.MemoryManager used on opensource projects

Example 6 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BatchTask method initLocalStrategies.

/**
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 */
protected void initLocalStrategies(int numInputs) throws Exception {
    final MemoryManager memMan = getMemoryManager();
    final IOManager ioMan = getIOManager();
    this.localStrategies = new CloseableInputProvider<?>[numInputs];
    this.inputs = new MutableObjectIterator<?>[numInputs];
    this.excludeFromReset = new boolean[numInputs];
    this.inputIsCached = new boolean[numInputs];
    this.inputIsAsyncMaterialized = new boolean[numInputs];
    this.materializationMemory = new int[numInputs];
    // created
    for (int i = 0; i < numInputs; i++) {
        initInputLocalStrategy(i);
    }
    // we do another loop over the inputs, because we want to instantiate all
    // sorters, etc before requesting the first input (as this call may block)
    // we have two types of materialized inputs, and both are replayable (can act as a cache)
    // The first variant materializes in a different thread and hence
    // acts as a pipeline breaker. this one should only be there, if a pipeline breaker is
    // needed.
    // the second variant spills to the side and will not read unless the result is also
    // consumed
    // in a pipelined fashion.
    this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
    this.tempBarriers = new TempBarrier<?>[numInputs];
    for (int i = 0; i < numInputs; i++) {
        final int memoryPages;
        final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
        final boolean cached = this.config.isInputCached(i);
        this.inputIsAsyncMaterialized[i] = async;
        this.inputIsCached[i] = cached;
        if (async || cached) {
            memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
            if (memoryPages <= 0) {
                throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
            }
            this.materializationMemory[i] = memoryPages;
        } else {
            memoryPages = 0;
        }
        if (async) {
            @SuppressWarnings({ "unchecked", "rawtypes" }) TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages, emptyList());
            barrier.startReading();
            this.tempBarriers[i] = barrier;
            this.inputs[i] = null;
        } else if (cached) {
            @SuppressWarnings({ "unchecked", "rawtypes" }) SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
            this.resettableInputs[i] = iter;
            this.inputs[i] = iter;
        }
    }
}

Example 7 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BatchTask method resetAllInputs.

protected void resetAllInputs() throws Exception {
    // NOTE: we need to do this before closing the local strategies
    for (int i = 0; i < this.inputs.length; i++) {
        if (this.inputIsCached[i] && this.resettableInputs[i] != null) {
            this.resettableInputs[i].consumeAndCacheRemainingData();
        }
    }
    // read them now and their data is cached
    for (int i = 0; i < this.localStrategies.length; i++) {
        if (this.localStrategies[i] != null) {
            this.localStrategies[i].close();
            this.localStrategies[i] = null;
        }
    }
    final MemoryManager memMan = getMemoryManager();
    final IOManager ioMan = getIOManager();
    // reset the caches, or re-run the input local strategy
    for (int i = 0; i < this.inputs.length; i++) {
        if (this.excludeFromReset[i]) {
            if (this.tempBarriers[i] != null) {
                this.tempBarriers[i].close();
                this.tempBarriers[i] = null;
            } else if (this.resettableInputs[i] != null) {
                this.resettableInputs[i].close();
                this.resettableInputs[i] = null;
            }
        } else {
            // make sure the input is not available directly, but are lazily fetched again
            this.inputs[i] = null;
            if (this.inputIsCached[i]) {
                if (this.tempBarriers[i] != null) {
                    this.inputs[i] = this.tempBarriers[i].getIterator();
                } else if (this.resettableInputs[i] != null) {
                    this.resettableInputs[i].reset();
                    this.inputs[i] = this.resettableInputs[i];
                } else {
                    throw new RuntimeException("Found a resettable input, but no temp barrier and no resettable iterator.");
                }
            } else {
                // close the async barrier if there is one
                List<MemorySegment> allocated = tempBarriers[i] == null ? emptyList() : tempBarriers[i].closeAndGetLeftoverMemory();
                tempBarriers[i] = null;
                try {
                    initInputLocalStrategy(i);
                    if (this.inputIsAsyncMaterialized[i]) {
                        final int pages = this.materializationMemory[i];
                        Preconditions.checkState(allocated.size() <= // pages shouldn't change, but some segments
                        pages);
                        // might have been consumed
                        @SuppressWarnings({ "unchecked", "rawtypes" }) TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, pages, allocated);
                        barrier.startReading();
                        this.tempBarriers[i] = barrier;
                        this.inputs[i] = null;
                    } else {
                        memMan.release(allocated);
                    }
                } catch (Exception exception) {
                    try {
                        memMan.release(allocated);
                    } catch (Exception releaseException) {
                        exception.addSuppressed(releaseException);
                    }
                    throw exception;
                }
            }
        }
    }
}

Example 8 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class JoinDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    // obtain task manager's memory manager and I/O manager
    final MemoryManager memoryManager = this.taskContext.getMemoryManager();
    final IOManager ioManager = this.taskContext.getIOManager();
    // set up memory and I/O parameters
    final double fractionAvailableMemory = config.getRelativeMemoryDriver();
    final int numPages = memoryManager.computeNumberOfPages(fractionAvailableMemory);
    // test minimum memory requirements
    final DriverStrategy ls = config.getDriverStrategy();
    final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
    final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
    // get the key positions and types
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
    final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
        throw new Exception("Missing pair comparator factory for join driver");
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
    boolean hashJoinUseBitMaps = taskContext.getTaskManagerInfo().getConfiguration().getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
    // create and return joining iterator according to provided local strategy.
    if (objectReuseEnabled) {
        switch(ls) {
            case INNER_MERGE:
                this.joinIterator = new ReusingMergeInnerJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, numPages, this.taskContext.getContainingTask());
                break;
            case HYBRIDHASH_BUILD_FIRST:
                this.joinIterator = new ReusingBuildFirstHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator21(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            case HYBRIDHASH_BUILD_SECOND:
                this.joinIterator = new ReusingBuildSecondHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            default:
                throw new Exception("Unsupported driver strategy for join driver: " + ls.name());
        }
    } else {
        switch(ls) {
            case INNER_MERGE:
                this.joinIterator = new NonReusingMergeInnerJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, numPages, this.taskContext.getContainingTask());
                break;
            case HYBRIDHASH_BUILD_FIRST:
                this.joinIterator = new NonReusingBuildFirstHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator21(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            case HYBRIDHASH_BUILD_SECOND:
                this.joinIterator = new NonReusingBuildSecondHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            default:
                throw new Exception("Unsupported driver strategy for join driver: " + ls.name());
        }
    }
    // open the iterator - this triggers the sorting or hash-table building
    // and blocks until the iterator is ready
    this.joinIterator.open();
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("join task iterator ready."));
    }
}

Example 9 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class SynchronousChainedCombineDriver method openTask.

@Override
public void openTask() throws Exception {
    // open the stub first
    final Configuration stubConfig = this.config.getStubParameters();
    BatchTask.openUserCode(this.combiner, stubConfig);
    // ----------------- Set up the sorter -------------------------
    // instantiate the serializer / comparator
    final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
    final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
    final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
    this.serializer = serializerFactory.getSerializer();
    TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
    this.groupingComparator = groupingComparatorFactory.createComparator();
    MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
    final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
    this.memory = memManager.allocatePages(this.parent, numMemoryPages);
    // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
    if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
        this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
    } else {
        this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
}

Example 10 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class ChainedReduceCombineDriver method openTask.

@Override
public void openTask() throws Exception {
    // open the stub first
    final Configuration stubConfig = config.getStubParameters();
    BatchTask.openUserCode(reducer, stubConfig);
    // instantiate the serializer / comparator
    serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer();
    comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator();
    MemoryManager memManager = parent.getEnvironment().getMemoryManager();
    final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
    memory = memManager.allocatePages(parent, numMemoryPages);
    LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    switch(strategy) {
        case SORTED_PARTIAL_REDUCE:
            // sorter
            if (comparator.supportsSerializationWithKeyNormalization() && serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
                sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
            } else {
                sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
            }
            break;
        case HASHED_PARTIAL_REDUCE:
            table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
            table.open();
            reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled);
            break;
    }
}