Examples with DiskRangeList org.apache.hadoop.hive.common.io.DiskRangeList used on opensource projects

Example 6 with DiskRangeList

use of org.apache.hadoop.hive.common.io.DiskRangeList in project hive by apache.

the class EncodedReaderImpl method getDataFromCacheAndDisk.

private DiskRangeList.MutateHelper getDataFromCacheAndDisk(DiskRangeList listToRead, long stripeOffset, boolean hasFileId, IdentityHashMap<ByteBuffer, Boolean> toRelease) throws IOException {
    DiskRangeList.MutateHelper toRead = new DiskRangeList.MutateHelper(listToRead);
    if (LOG.isInfoEnabled()) {
        LOG.info("Resulting disk ranges to read (file " + fileKey + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next));
    }
    BooleanRef isAllInCache = new BooleanRef();
    if (hasFileId) {
        cacheWrapper.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY, isAllInCache);
        if (LOG.isInfoEnabled()) {
            LOG.info("Disk ranges after cache (found everything " + isAllInCache.value + "; file " + fileKey + ", base offset " + stripeOffset + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next));
        }
        trace.logRanges(fileKey, stripeOffset, toRead.next, RangesSrc.CACHE);
    }
    // can be freed in advance, we remove it from the map.
    if (!isAllInCache.value) {
        boolean hasError = true;
        try {
            if (!isDataReaderOpen) {
                this.dataReader.open();
                isDataReaderOpen = true;
            }
            dataReader.readFileData(toRead.next, stripeOffset, cacheWrapper.getAllocator().isDirectAlloc());
            toRelease = new IdentityHashMap<>();
            DiskRangeList drl = toRead.next;
            while (drl != null) {
                if (drl instanceof BufferChunk) {
                    toRelease.put(drl.getData(), true);
                }
                drl = drl.next;
            }
            hasError = false;
        } finally {
            // We are assuming here that toRelease will not be present in such cases.
            if (hasError) {
                releaseInitialRefcounts(toRead.next);
            }
        }
    }
    return toRead;
}

Example 7 with DiskRangeList

use of org.apache.hadoop.hive.common.io.DiskRangeList in project hive by apache.

the class EncodedReaderImpl method prepareRangesForUncompressedRead.

/**
 * Subset of readEncodedStream specific to uncompressed streams, separate to avoid long methods.
 */
private CacheChunk prepareRangesForUncompressedRead(long cOffset, long endCOffset, long streamOffset, long unlockUntilCOffset, DiskRangeList current, ColumnStreamData columnStreamData) throws IOException {
    // Note: we are called after preReadUncompressedStream, so it doesn't have to do nearly as much
    // as prepareRangesForCompressedRead does; e.g. every buffer is already a CacheChunk.
    long currentOffset = cOffset;
    CacheChunk lastUncompressed = null;
    boolean isFirst = true;
    while (true) {
        DiskRangeList next = null;
        assert current instanceof CacheChunk;
        lastUncompressed = (CacheChunk) current;
        if (isTracingEnabled) {
            LOG.trace("Locking " + lastUncompressed.getBuffer() + " due to reuse");
        }
        cacheWrapper.reuseBuffer(lastUncompressed.getBuffer());
        if (isFirst) {
            columnStreamData.setIndexBaseOffset((int) (lastUncompressed.getOffset() - streamOffset));
            isFirst = false;
        }
        columnStreamData.getCacheBuffers().add(lastUncompressed.getBuffer());
        currentOffset = lastUncompressed.getEnd();
        if (isTracingEnabled) {
            LOG.trace("Adding an uncompressed buffer " + lastUncompressed.getBuffer());
        }
        ponderReleaseInitialRefcount(unlockUntilCOffset, streamOffset, lastUncompressed);
        next = current.next;
        if (next == null || (endCOffset >= 0 && currentOffset >= endCOffset)) {
            break;
        }
        current = next;
    }
    return lastUncompressed;
}

Example 8 with DiskRangeList

use of org.apache.hadoop.hive.common.io.DiskRangeList in project hive by apache.

the class EncodedReaderImpl method readEncodedStream.

/**
 * Uncompresses part of the stream. RGs can overlap, so we cannot just go and decompress
 * and remove what we have returned. We will keep iterator as a "hint" point.
 * @param baseOffset Absolute offset of boundaries and ranges relative to file, for cache keys.
 * @param start Ordered ranges containing file data. Helpful if they point close to cOffset.
 * @param cOffset Start offset to decompress.
 * @param endCOffset End offset to decompress; estimate, partial CBs will be ignored.
 * @param csd Stream data, to add the results.
 * @param unlockUntilCOffset The offset until which the buffers can be unlocked in cache, as
 *                           they will not be used in future calls (see the class comment in
 *                           EncodedReaderImpl about refcounts).
 * @return Last buffer cached during decompression. Cache buffers are never removed from
 *         the master list, so they are safe to keep as iterators for various streams.
 */
public DiskRangeList readEncodedStream(long baseOffset, DiskRangeList start, long cOffset, long endCOffset, ColumnStreamData csd, long unlockUntilCOffset, long streamOffset, IdentityHashMap<ByteBuffer, Boolean> toRelease) throws IOException {
    if (csd.getCacheBuffers() == null) {
        csd.setCacheBuffers(new ArrayList<MemoryBuffer>());
    } else {
        csd.getCacheBuffers().clear();
    }
    if (cOffset == endCOffset)
        return null;
    List<ProcCacheChunk> toDecompress = null;
    List<IncompleteCb> badEstimates = null;
    List<ByteBuffer> toReleaseCopies = null;
    if (isCompressed) {
        toReleaseCopies = new ArrayList<>();
        toDecompress = new ArrayList<>();
        badEstimates = new ArrayList<>();
    }
    // 1. Find our bearings in the stream. Normally, iter will already point either to where we
    // want to be, or just before. However, RGs can overlap due to encoding, so we may have
    // to return to a previous block.
    DiskRangeList current = findExactPosition(start, cOffset);
    if (isTracingEnabled) {
        LOG.trace("Starting read for [" + cOffset + "," + endCOffset + ") at " + current);
    }
    trace.logStartRead(current);
    CacheChunk lastUncompressed = null;
    // 2. Go thru the blocks; add stuff to results and prepare the decompression work (see below).
    try {
        lastUncompressed = isCompressed ? prepareRangesForCompressedRead(cOffset, endCOffset, streamOffset, unlockUntilCOffset, current, csd, toRelease, toReleaseCopies, toDecompress, badEstimates) : prepareRangesForUncompressedRead(cOffset, endCOffset, streamOffset, unlockUntilCOffset, current, csd);
    } catch (Exception ex) {
        LOG.error("Failed " + (isCompressed ? "" : "un") + "compressed read; cOffset " + cOffset + ", endCOffset " + endCOffset + ", streamOffset " + streamOffset + ", unlockUntilCOffset " + unlockUntilCOffset + "; ranges passed in " + RecordReaderUtils.stringifyDiskRanges(start) + "; ranges passed to prepare " + // Don't log exception here.
        RecordReaderUtils.stringifyDiskRanges(current));
        throw (ex instanceof IOException) ? (IOException) ex : new IOException(ex);
    }
    // 2.5. Remember the bad estimates for future reference.
    if (badEstimates != null && !badEstimates.isEmpty()) {
        // Relies on the fact that cache does not actually store these.
        DiskRange[] cacheKeys = badEstimates.toArray(new DiskRange[badEstimates.size()]);
        long[] result = cacheWrapper.putFileData(fileKey, cacheKeys, null, baseOffset, tag);
        // We don't expect conflicts from bad estimates.
        assert result == null;
    }
    if (toDecompress == null || toDecompress.isEmpty()) {
        releaseBuffers(toReleaseCopies, false);
        // Nothing to do.
        return lastUncompressed;
    }
    // 3. Allocate the buffers, prepare cache keys.
    // At this point, we have read all the CBs we need to read. cacheBuffers contains some cache
    // data and some unallocated membufs for decompression. toDecompress contains all the work we
    // need to do, and each item points to one of the membufs in cacheBuffers as target. The iter
    // has also been adjusted to point to these buffers instead of compressed data for the ranges.
    MemoryBuffer[] targetBuffers = new MemoryBuffer[toDecompress.size()];
    DiskRange[] cacheKeys = new DiskRange[toDecompress.size()];
    int ix = 0;
    for (ProcCacheChunk chunk : toDecompress) {
        // Relies on the fact that cache does not actually store these.
        cacheKeys[ix] = chunk;
        targetBuffers[ix] = chunk.getBuffer();
        ++ix;
    }
    cacheWrapper.getAllocator().allocateMultiple(targetBuffers, bufferSize, cacheWrapper.getDataBufferFactory());
    // 4. Now decompress (or copy) the data into cache buffers.
    for (ProcCacheChunk chunk : toDecompress) {
        ByteBuffer dest = chunk.getBuffer().getByteBufferRaw();
        if (chunk.isOriginalDataCompressed) {
            boolean isOk = false;
            try {
                decompressChunk(chunk.originalData, codec, dest);
                isOk = true;
            } finally {
                if (!isOk) {
                    isCodecFailure = true;
                }
            }
        } else {
            copyUncompressedChunk(chunk.originalData, dest);
        }
        if (isTracingEnabled) {
            LOG.trace("Locking " + chunk.getBuffer() + " due to reuse (after decompression)");
        }
        // Note that this assumes the failure during incref means incref didn't occur.
        try {
            cacheWrapper.reuseBuffer(chunk.getBuffer());
        } finally {
            chunk.originalData = null;
        }
    }
    // 5. Release the copies we made directly to the cleaner.
    releaseBuffers(toReleaseCopies, false);
    // 6. Finally, put uncompressed data to cache.
    if (fileKey != null) {
        long[] collisionMask = cacheWrapper.putFileData(fileKey, cacheKeys, targetBuffers, baseOffset, tag);
        processCacheCollisions(collisionMask, toDecompress, targetBuffers, csd.getCacheBuffers());
    }
    // Release initial refcounts.
    for (ProcCacheChunk chunk : toDecompress) {
        ponderReleaseInitialRefcount(unlockUntilCOffset, streamOffset, chunk);
    }
    return lastUncompressed;
}

Example 9 with DiskRangeList

use of org.apache.hadoop.hive.common.io.DiskRangeList in project hive by apache.

the class EncodedReaderImpl method copyAndReplaceUncompressedChunks.

private void copyAndReplaceUncompressedChunks(UncompressedCacheChunk candidateCached, ByteBuffer dest, CacheChunk tcc, boolean isValid) {
    int startPos = dest.position(), startLim = dest.limit();
    DiskRangeList next = null;
    for (int i = 0; i < candidateCached.getCount(); ++i) {
        BufferChunk chunk = (i == 0) ? candidateCached.getChunk() : (BufferChunk) next;
        dest.put(chunk.getData());
        if (isValid) {
            trace.logValidUncompresseedChunk(startLim - startPos, chunk);
        }
        next = chunk.next;
        if (i == 0) {
            chunk.replaceSelfWith(tcc);
        } else {
            chunk.removeSelf();
        }
    }
    int newPos = dest.position();
    if (newPos > startLim) {
        throw new AssertionError("After copying, buffer [" + startPos + ", " + startLim + ") became [" + newPos + ", " + dest.limit() + ")");
    }
    dest.position(startPos);
    dest.limit(newPos);
}

Example 10 with DiskRangeList

use of org.apache.hadoop.hive.common.io.DiskRangeList in project hive by apache.

the class OrcEncodedDataReader method getStripeFooterFromCacheOrDisk.

private OrcProto.StripeFooter getStripeFooterFromCacheOrDisk(StripeInformation si, OrcBatchKey stripeKey) throws IOException {
    boolean hasCache = fileKey != null && metadataCache != null;
    if (hasCache) {
        LlapBufferOrBuffers footerBuffers = metadataCache.getStripeTail(stripeKey);
        if (footerBuffers != null) {
            try {
                counters.incrCounter(LlapIOCounters.METADATA_CACHE_HIT);
                ensureCodecFromFileMetadata();
                MemoryBuffer footerBuffer = footerBuffers.getSingleBuffer();
                if (footerBuffer != null) {
                    ByteBuffer bb = footerBuffer.getByteBufferDup();
                    return buildStripeFooter(Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), bb.remaining(), codec, fileMetadata.getCompressionBufferSize());
                } else {
                    MemoryBuffer[] footerBufferArray = footerBuffers.getMultipleBuffers();
                    int pos = 0;
                    List<DiskRange> bcs = new ArrayList<>(footerBufferArray.length);
                    for (MemoryBuffer buf : footerBufferArray) {
                        ByteBuffer bb = buf.getByteBufferDup();
                        bcs.add(new BufferChunk(bb, pos));
                        pos += bb.remaining();
                    }
                    return buildStripeFooter(bcs, pos, codec, fileMetadata.getCompressionBufferSize());
                }
            } finally {
                metadataCache.decRefBuffer(footerBuffers);
            }
        }
        counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS);
    }
    long offset = si.getOffset() + si.getIndexLength() + si.getDataLength();
    long startTime = counters.startTimeCounter();
    ensureRawDataReader(true);
    // TODO: add this to metadatareader in ORC - SI => metadata buffer, not just metadata.
    if (LOG.isTraceEnabled()) {
        LOG.trace("Reading [" + offset + ", " + (offset + si.getFooterLength()) + ") based on " + si);
    }
    DiskRangeList footerRange = rawDataReader.readFileData(new DiskRangeList(offset, offset + si.getFooterLength()), 0, false);
    // LOG.error("Got " + RecordReaderUtils.stringifyDiskRanges(footerRange));
    counters.incrTimeCounter(LlapIOCounters.HDFS_TIME_NS, startTime);
    // Can only happens w/zcr for a single input buffer.
    assert footerRange.next == null;
    if (hasCache) {
        LlapBufferOrBuffers cacheBuf = metadataCache.putStripeTail(stripeKey, footerRange.getData().duplicate(), cacheTag);
        // We don't use this one.
        metadataCache.decRefBuffer(cacheBuf);
    }
    ByteBuffer bb = footerRange.getData().duplicate();
    CompressionKind kind = orcReader.getCompressionKind();
    boolean isPool = useCodecPool;
    CompressionCodec codec = isPool ? OrcCodecPool.getCodec(kind) : WriterImpl.createCodec(kind);
    boolean isCodecError = true;
    try {
        OrcProto.StripeFooter result = buildStripeFooter(Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), bb.remaining(), codec, orcReader.getCompressionSize());
        isCodecError = false;
        return result;
    } finally {
        try {
            if (isPool && !isCodecError) {
                OrcCodecPool.returnCodec(kind, codec);
            } else {
                codec.close();
            }
        } catch (Exception ex) {
            LOG.error("Ignoring codec cleanup error", ex);
        }
    }
}