Examples with StoreFileReader org.apache.hadoop.hbase.regionserver.StoreFileReader used on opensource projects

Example 1 with StoreFileReader

use of org.apache.hadoop.hbase.regionserver.StoreFileReader in project hbase by apache.

the class Compactor method getFileDetails.

/**
   * Extracts some details about the files to compact that are commonly needed by compactors.
   * @param filesToCompact Files.
   * @param allFiles Whether all files are included for compaction
   * @return The result.
   */
protected FileDetails getFileDetails(Collection<StoreFile> filesToCompact, boolean allFiles) throws IOException {
    FileDetails fd = new FileDetails();
    long oldestHFileTimeStampToKeepMVCC = System.currentTimeMillis() - (1000L * 60 * 60 * 24 * this.keepSeqIdPeriod);
    for (StoreFile file : filesToCompact) {
        if (allFiles && (file.getModificationTimeStamp() < oldestHFileTimeStampToKeepMVCC)) {
            // MVCC value to keep
            if (fd.minSeqIdToKeep < file.getMaxMemstoreTS()) {
                fd.minSeqIdToKeep = file.getMaxMemstoreTS();
            }
        }
        long seqNum = file.getMaxSequenceId();
        fd.maxSeqId = Math.max(fd.maxSeqId, seqNum);
        StoreFileReader r = file.getReader();
        if (r == null) {
            LOG.warn("Null reader for " + file.getPath());
            continue;
        }
        // NOTE: use getEntries when compacting instead of getFilterEntries, otherwise under-sized
        // blooms can cause progress to be miscalculated or if the user switches bloom
        // type (e.g. from ROW to ROWCOL)
        long keyCount = r.getEntries();
        fd.maxKeyCount += keyCount;
        // calculate the latest MVCC readpoint in any of the involved store files
        Map<byte[], byte[]> fileInfo = r.loadFileInfo();
        byte[] tmp = null;
        // SeqId number.
        if (r.isBulkLoaded()) {
            fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, r.getSequenceID());
        } else {
            tmp = fileInfo.get(HFile.Writer.MAX_MEMSTORE_TS_KEY);
            if (tmp != null) {
                fd.maxMVCCReadpoint = Math.max(fd.maxMVCCReadpoint, Bytes.toLong(tmp));
            }
        }
        tmp = fileInfo.get(FileInfo.MAX_TAGS_LEN);
        if (tmp != null) {
            fd.maxTagsLength = Math.max(fd.maxTagsLength, Bytes.toInt(tmp));
        }
        // If required, calculate the earliest put timestamp of all involved storefiles.
        // This is used to remove family delete marker during compaction.
        long earliestPutTs = 0;
        if (allFiles) {
            tmp = fileInfo.get(StoreFile.EARLIEST_PUT_TS);
            if (tmp == null) {
                // There's a file with no information, must be an old one
                // assume we have very old puts
                fd.earliestPutTs = earliestPutTs = HConstants.OLDEST_TIMESTAMP;
            } else {
                earliestPutTs = Bytes.toLong(tmp);
                fd.earliestPutTs = Math.min(fd.earliestPutTs, earliestPutTs);
            }
        }
        tmp = fileInfo.get(StoreFile.TIMERANGE_KEY);
        TimeRangeTracker trt = TimeRangeTracker.getTimeRangeTracker(tmp);
        fd.latestPutTs = trt == null ? HConstants.LATEST_TIMESTAMP : trt.getMax();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Compacting " + file + ", keycount=" + keyCount + ", bloomtype=" + r.getBloomFilterType().toString() + ", size=" + TraditionalBinaryPrefix.long2String(r.length(), "", 1) + ", encoding=" + r.getHFileReader().getDataBlockEncoding() + ", seqNum=" + seqNum + (allFiles ? ", earliestPutTs=" + earliestPutTs : ""));
        }
    }
    return fd;
}

Example 2 with StoreFileReader

use of org.apache.hadoop.hbase.regionserver.StoreFileReader in project hbase by apache.

the class MockStoreFileGenerator method createMockStoreFile.

protected StoreFile createMockStoreFile(final long sizeInBytes, final long seqId) {
    StoreFile mockSf = mock(StoreFile.class);
    StoreFileReader reader = mock(StoreFileReader.class);
    String stringPath = "/hbase/testTable/regionA/" + RandomStringUtils.random(FILENAME_LENGTH, 0, 0, true, true, null, random);
    Path path = new Path(stringPath);
    when(reader.getSequenceID()).thenReturn(seqId);
    when(reader.getTotalUncompressedBytes()).thenReturn(sizeInBytes);
    when(reader.length()).thenReturn(sizeInBytes);
    when(mockSf.getPath()).thenReturn(path);
    when(mockSf.excludeFromMinorCompaction()).thenReturn(false);
    // TODO come back to
    when(mockSf.isReference()).thenReturn(false);
    // this when selection takes this into account
    when(mockSf.getReader()).thenReturn(reader);
    String toString = Objects.toStringHelper("MockStoreFile").add("isReference", false).add("fileSize", StringUtils.humanReadableInt(sizeInBytes)).add("seqId", seqId).add("path", stringPath).toString();
    when(mockSf.toString()).thenReturn(toString);
    return mockSf;
}

Example 3 with StoreFileReader

use of org.apache.hadoop.hbase.regionserver.StoreFileReader in project hbase by apache.

the class TestCompactor method createDummyStoreFile.

public static StoreFile createDummyStoreFile(long maxSequenceId) throws Exception {
    // "Files" are totally unused, it's Scanner class below that gives compactor fake KVs.
    // But compaction depends on everything under the sun, so stub everything with dummies.
    StoreFile sf = mock(StoreFile.class);
    StoreFileReader r = mock(StoreFileReader.class);
    when(r.length()).thenReturn(1L);
    when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
    when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
    when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class));
    when(sf.getReader()).thenReturn(r);
    when(sf.createReader()).thenReturn(r);
    when(sf.createReader(anyBoolean())).thenReturn(r);
    when(sf.cloneForReader()).thenReturn(sf);
    when(sf.getMaxSequenceId()).thenReturn(maxSequenceId);
    return sf;
}

Example 4 with StoreFileReader

use of org.apache.hadoop.hbase.regionserver.StoreFileReader in project hbase by apache.

the class CompactionRequest method recalculateSize.

/**
   * Recalculate the size of the compaction based on current files.
   * @param files files that should be included in the compaction
   */
private void recalculateSize() {
    long sz = 0;
    for (StoreFile sf : this.filesToCompact) {
        StoreFileReader r = sf.getReader();
        sz += r == null ? 0 : r.length();
    }
    this.totalSize = sz;
}

Example 5 with StoreFileReader

use of org.apache.hadoop.hbase.regionserver.StoreFileReader in project hbase by apache.

the class TestStripeCompactionPolicy method createFile.

private static StoreFile createFile(long size) throws Exception {
    StoreFile sf = mock(StoreFile.class);
    when(sf.getPath()).thenReturn(new Path("moo"));
    StoreFileReader r = mock(StoreFileReader.class);
    when(r.getEntries()).thenReturn(size);
    when(r.length()).thenReturn(size);
    when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
    when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
    when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class));
    when(sf.getReader()).thenReturn(r);
    when(sf.createReader(anyBoolean())).thenReturn(r);
    when(sf.createReader()).thenReturn(r);
    when(sf.cloneForReader()).thenReturn(sf);
    return sf;
}