Examples with BulkBlockChannelReader org.apache.flink.runtime.io.disk.iomanager.BulkBlockChannelReader used on opensource projects

Example 1 with BulkBlockChannelReader

use of org.apache.flink.runtime.io.disk.iomanager.BulkBlockChannelReader in project flink by apache.

the class ReOpenableHashPartition method restorePartitionBuffers.

/**
	 * This method is called every time a multi-match hash map is opened again for a new probe input.
	 * @param ioManager 
	 * @param availableMemory 
	 * @throws IOException 
	 */
void restorePartitionBuffers(IOManager ioManager, List<MemorySegment> availableMemory) throws IOException {
    final BulkBlockChannelReader reader = ioManager.createBulkBlockChannelReader(this.initialBuildSideChannel, availableMemory, this.initialPartitionBuffersCount);
    reader.close();
    final List<MemorySegment> partitionBuffersFromDisk = reader.getFullSegments();
    this.partitionBuffers = (MemorySegment[]) partitionBuffersFromDisk.toArray(new MemorySegment[partitionBuffersFromDisk.size()]);
    this.overflowSegments = new MemorySegment[2];
    this.numOverflowSegments = 0;
    this.nextOverflowBucket = 0;
    this.isRestored = true;
}

Example 2 with BulkBlockChannelReader

use of org.apache.flink.runtime.io.disk.iomanager.BulkBlockChannelReader in project flink by apache.

the class MutableHashTable method buildTableFromSpilledPartition.

protected void buildTableFromSpilledPartition(final HashPartition<BT, PT> p) throws IOException {
    final int nextRecursionLevel = p.getRecursionLevel() + 1;
    if (nextRecursionLevel > MAX_RECURSION_DEPTH) {
        throw new RuntimeException("Hash join exceeded maximum number of recursions, without reducing " + "partitions enough to be memory resident. Probably cause: Too many duplicate keys.");
    }
    // we distinguish two cases here:
    // 1) The partition fits entirely into main memory. That is the case if we have enough buffers for
    //    all partition segments, plus enough buffers to hold the table structure.
    //    --> We read the partition in as it is and create a hashtable that references only
    //        that single partition.
    // 2) We can not guarantee that enough memory segments are available and read the partition
    //    in, distributing its data among newly created partitions.
    final int totalBuffersAvailable = this.availableMemory.size() + this.writeBehindBuffersAvailable;
    if (totalBuffersAvailable != this.totalNumBuffers - this.numWriteBehindBuffers) {
        throw new RuntimeException("Hash Join bug in memory management: Memory buffers leaked.");
    }
    long numBuckets = p.getBuildSideRecordCount() / NUM_ENTRIES_PER_BUCKET + 1;
    // we need to consider the worst case where everything hashes to one bucket which needs to overflow by the same
    // number of total buckets again. Also, one buffer needs to remain for the probing
    final long totalBuffersNeeded = 2 * (numBuckets / (this.bucketsPerSegmentMask + 1)) + p.getBuildSideBlockCount() + 2;
    if (totalBuffersNeeded < totalBuffersAvailable) {
        // we are guaranteed to stay in memory
        ensureNumBuffersReturned(p.getBuildSideBlockCount());
        // first read the partition in
        final BulkBlockChannelReader reader = this.ioManager.createBulkBlockChannelReader(p.getBuildSideChannel().getChannelID(), this.availableMemory, p.getBuildSideBlockCount());
        // call waits until all is read
        if (keepBuildSidePartitions && p.recursionLevel == 0) {
            // keep the partitions
            reader.close();
        } else {
            reader.closeAndDelete();
        }
        final List<MemorySegment> partitionBuffers = reader.getFullSegments();
        final HashPartition<BT, PT> newPart = new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer, 0, nextRecursionLevel, partitionBuffers, p.getBuildSideRecordCount(), this.segmentSize, p.getLastSegmentLimit());
        this.partitionsBeingBuilt.add(newPart);
        // erect the buckets
        initTable((int) numBuckets, (byte) 1);
        // now, index the partition through a hash table
        final HashPartition<BT, PT>.PartitionIterator<BT, PT> pIter = newPart.getPartitionIterator(this.buildSideComparator);
        BT record = this.buildSideSerializer.createInstance();
        while ((record = pIter.next(record)) != null) {
            final int hashCode = hash(pIter.getCurrentHashCode(), nextRecursionLevel);
            final int posHashCode = hashCode % this.numBuckets;
            final long pointer = pIter.getPointer();
            // get the bucket for the given hash code
            final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits;
            final int bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
            final MemorySegment bucket = this.buckets[bucketArrayPos];
            insertBucketEntry(newPart, bucket, bucketInSegmentPos, hashCode, pointer, false);
        }
    } else {
        // we need to partition and partially spill
        final int avgRecordLenPartition = (int) (((long) p.getBuildSideBlockCount()) * this.segmentSize / p.getBuildSideRecordCount());
        final int bucketCount = getInitialTableSize(totalBuffersAvailable, this.segmentSize, getPartitioningFanOutNoEstimates(totalBuffersAvailable), avgRecordLenPartition);
        // compute in how many splits, we'd need to partition the result 
        final int splits = (int) (totalBuffersNeeded / totalBuffersAvailable) + 1;
        final int partitionFanOut = Math.min(10 * splits, /* being conservative */
        MAX_NUM_PARTITIONS);
        createPartitions(partitionFanOut, nextRecursionLevel);
        // set up the table structure. the write behind buffers are taken away, as are one buffer per partition
        initTable(bucketCount, (byte) partitionFanOut);
        // go over the complete input and insert every element into the hash table
        // first set up the reader with some memory.
        final List<MemorySegment> segments = new ArrayList<MemorySegment>(2);
        segments.add(getNextBuffer());
        segments.add(getNextBuffer());
        final BlockChannelReader<MemorySegment> inReader = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID());
        final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(inReader, segments, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false);
        final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView, this.availableMemory, this.buildSideSerializer);
        final TypeComparator<BT> btComparator = this.buildSideComparator;
        BT rec = this.buildSideSerializer.createInstance();
        while ((rec = inIter.next(rec)) != null) {
            final int hashCode = hash(btComparator.hash(rec), nextRecursionLevel);
            insertIntoTable(rec, hashCode);
        }
        if (keepBuildSidePartitions && p.recursionLevel == 0) {
            // keep the partitions
            inReader.close();
        } else {
            inReader.closeAndDelete();
        }
        // finalize the partitions
        for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) {
            HashPartition<BT, PT> part = this.partitionsBeingBuilt.get(i);
            part.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
        }
    }
}