Examples with StoreFile org.apache.hadoop.hbase.regionserver.StoreFile used on opensource projects

Example 26 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class DateTieredCompactionPolicy method shouldPerformMajorCompaction.

public boolean shouldPerformMajorCompaction(final Collection<StoreFile> filesToCompact) throws IOException {
    long mcTime = getNextMajorCompactTime(filesToCompact);
    if (filesToCompact == null || mcTime == 0) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("filesToCompact: " + filesToCompact + " mcTime: " + mcTime);
        }
        return false;
    }
    // TODO: Use better method for determining stamp of last major (HBASE-2990)
    long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
    long now = EnvironmentEdgeManager.currentTime();
    if (lowTimestamp <= 0L || lowTimestamp >= (now - mcTime)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("lowTimestamp: " + lowTimestamp + " lowTimestamp: " + lowTimestamp + " now: " + now + " mcTime: " + mcTime);
        }
        return false;
    }
    long cfTTL = this.storeConfigInfo.getStoreFileTtl();
    HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
    List<Long> boundaries = getCompactBoundariesForMajor(filesToCompact, now);
    boolean[] filesInWindow = new boolean[boundaries.size()];
    for (StoreFile file : filesToCompact) {
        Long minTimestamp = file.getMinimumTimestamp();
        long oldest = (minTimestamp == null) ? Long.MIN_VALUE : now - minTimestamp.longValue();
        if (cfTTL != Long.MAX_VALUE && oldest >= cfTTL) {
            LOG.debug("Major compaction triggered on store " + this + "; for TTL maintenance");
            return true;
        }
        if (!file.isMajorCompaction() || file.isBulkLoadResult()) {
            LOG.debug("Major compaction triggered on store " + this + ", because there are new files and time since last major compaction " + (now - lowTimestamp) + "ms");
            return true;
        }
        int lowerWindowIndex = Collections.binarySearch(boundaries, minTimestamp == null ? (Long) Long.MAX_VALUE : minTimestamp);
        int upperWindowIndex = Collections.binarySearch(boundaries, file.getMaximumTimestamp() == null ? (Long) Long.MAX_VALUE : file.getMaximumTimestamp());
        // Handle boundary conditions and negative values of binarySearch
        lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex;
        upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex;
        if (lowerWindowIndex != upperWindowIndex) {
            LOG.debug("Major compaction triggered on store " + this + "; because file " + file.getPath() + " has data with timestamps cross window boundaries");
            return true;
        } else if (filesInWindow[upperWindowIndex]) {
            LOG.debug("Major compaction triggered on store " + this + "; because there are more than one file in some windows");
            return true;
        } else {
            filesInWindow[upperWindowIndex] = true;
        }
        hdfsBlocksDistribution.add(file.getHDFSBlockDistribution());
    }
    float blockLocalityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(RSRpcServices.getHostname(comConf.conf, false));
    if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) {
        LOG.debug("Major compaction triggered on store " + this + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")");
        return true;
    }
    LOG.debug("Skipping major compaction of " + this + ", because the files are already major compacted");
    return false;
}

Example 27 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class DateTieredCompactionPolicy method selectMinorCompaction.

/**
   * We receive store files sorted in ascending order by seqId then scan the list of files. If the
   * current file has a maxTimestamp older than last known maximum, treat this file as it carries
   * the last known maximum. This way both seqId and timestamp are in the same order. If files carry
   * the same maxTimestamps, they are ordered by seqId. We then reverse the list so they are ordered
   * by seqId and maxTimestamp in descending order and build the time windows. All the out-of-order
   * data into the same compaction windows, guaranteeing contiguous compaction based on sequence id.
   */
public CompactionRequest selectMinorCompaction(ArrayList<StoreFile> candidateSelection, boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
    long now = EnvironmentEdgeManager.currentTime();
    long oldestToCompact = getOldestToCompact(comConf.getDateTieredMaxStoreFileAgeMillis(), now);
    List<Pair<StoreFile, Long>> storefileMaxTimestampPairs = Lists.newArrayListWithCapacity(candidateSelection.size());
    long maxTimestampSeen = Long.MIN_VALUE;
    for (StoreFile storeFile : candidateSelection) {
        // if there is out-of-order data,
        // we put them in the same window as the last file in increasing order
        maxTimestampSeen = Math.max(maxTimestampSeen, storeFile.getMaximumTimestamp() == null ? Long.MIN_VALUE : storeFile.getMaximumTimestamp());
        storefileMaxTimestampPairs.add(new Pair<>(storeFile, maxTimestampSeen));
    }
    Collections.reverse(storefileMaxTimestampPairs);
    CompactionWindow window = getIncomingWindow(now);
    int minThreshold = comConf.getDateTieredIncomingWindowMin();
    PeekingIterator<Pair<StoreFile, Long>> it = Iterators.peekingIterator(storefileMaxTimestampPairs.iterator());
    while (it.hasNext()) {
        if (window.compareToTimestamp(oldestToCompact) < 0) {
            break;
        }
        int compResult = window.compareToTimestamp(it.peek().getSecond());
        if (compResult > 0) {
            // If the file is too old for the window, switch to the next window
            window = window.nextEarlierWindow();
            minThreshold = comConf.getMinFilesToCompact();
        } else {
            // The file is within the target window
            ArrayList<StoreFile> fileList = Lists.newArrayList();
            // we tolerate files with future data although it is sub-optimal
            while (it.hasNext() && window.compareToTimestamp(it.peek().getSecond()) <= 0) {
                fileList.add(it.next().getFirst());
            }
            if (fileList.size() >= minThreshold) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Processing files: " + fileList + " for window: " + window);
                }
                DateTieredCompactionRequest request = generateCompactionRequest(fileList, window, mayUseOffPeak, mayBeStuck, minThreshold);
                if (request != null) {
                    return request;
                }
            }
        }
    }
    // A non-null file list is expected by HStore
    return new CompactionRequest(Collections.<StoreFile>emptyList());
}

Example 28 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class ExploringCompactionPolicy method filesInRatio.

/**
   * Check that all files satisfy the constraint
   *      FileSize(i) <= ( Sum(0,N,FileSize(_)) - FileSize(i) ) * Ratio.
   *
   * @param files List of store files to consider as a compaction candidate.
   * @param currentRatio The ratio to use.
   * @return a boolean if these files satisfy the ratio constraints.
   */
private boolean filesInRatio(final List<StoreFile> files, final double currentRatio) {
    if (files.size() < 2) {
        return true;
    }
    long totalFileSize = getTotalStoreSize(files);
    for (StoreFile file : files) {
        long singleFileSize = file.getReader().length();
        long sumAllOtherFileSizes = totalFileSize - singleFileSize;
        if (singleFileSize > sumAllOtherFileSizes * currentRatio) {
            return false;
        }
    }
    return true;
}

Example 29 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class RatioBasedCompactionPolicy method shouldPerformMajorCompaction.

/*
   * @param filesToCompact Files to compact. Can be null.
   * @return True if we should run a major compaction.
   */
@Override
public boolean shouldPerformMajorCompaction(final Collection<StoreFile> filesToCompact) throws IOException {
    boolean result = false;
    long mcTime = getNextMajorCompactTime(filesToCompact);
    if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) {
        return result;
    }
    // TODO: Use better method for determining stamp of last major (HBASE-2990)
    long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
    long now = EnvironmentEdgeManager.currentTime();
    if (lowTimestamp > 0L && lowTimestamp < (now - mcTime)) {
        // Major compaction time has elapsed.
        long cfTTL = this.storeConfigInfo.getStoreFileTtl();
        if (filesToCompact.size() == 1) {
            // Single file
            StoreFile sf = filesToCompact.iterator().next();
            Long minTimestamp = sf.getMinimumTimestamp();
            long oldest = (minTimestamp == null) ? Long.MIN_VALUE : now - minTimestamp.longValue();
            if (sf.isMajorCompaction() && (cfTTL == Long.MAX_VALUE || oldest < cfTTL)) {
                float blockLocalityIndex = sf.getHDFSBlockDistribution().getBlockLocalityIndex(RSRpcServices.getHostname(comConf.conf, false));
                if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) {
                    LOG.debug("Major compaction triggered on only store " + this + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")");
                    result = true;
                } else {
                    LOG.debug("Skipping major compaction of " + this + " because one (major) compacted file only, oldestTime " + oldest + "ms is < TTL=" + cfTTL + " and blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")");
                }
            } else if (cfTTL != HConstants.FOREVER && oldest > cfTTL) {
                LOG.debug("Major compaction triggered on store " + this + ", because keyvalues outdated; time since last major compaction " + (now - lowTimestamp) + "ms");
                result = true;
            }
        } else {
            LOG.debug("Major compaction triggered on store " + this + "; time since last major compaction " + (now - lowTimestamp) + "ms");
        }
        result = true;
    }
    return result;
}

Example 30 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class RatioBasedCompactionPolicy method applyCompactionPolicy.

/**
    * -- Default minor compaction selection algorithm:
    * choose CompactSelection from candidates --
    * First exclude bulk-load files if indicated in configuration.
    * Start at the oldest file and stop when you find the first file that
    * meets compaction criteria:
    * (1) a recently-flushed, small file (i.e. <= minCompactSize)
    * OR
    * (2) within the compactRatio of sum(newer_files)
    * Given normal skew, any newer files will also meet this criteria
    * <p/>
    * Additional Note:
    * If fileSizes.size() >> maxFilesToCompact, we will recurse on
    * compact().  Consider the oldest files first to avoid a
    * situation where we always compact [end-threshold,end).  Then, the
    * last file becomes an aggregate of the previous compactions.
    *
    * normal skew:
    *
    *         older ----> newer (increasing seqID)
    *     _
    *    | |   _
    *    | |  | |   _
    *  --|-|- |-|- |-|---_-------_-------  minCompactSize
    *    | |  | |  | |  | |  _  | |
    *    | |  | |  | |  | | | | | |
    *    | |  | |  | |  | | | | | |
    * @param candidates pre-filtrate
    * @return filtered subset
    */
protected ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates, boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
    if (candidates.isEmpty()) {
        return candidates;
    }
    // we're doing a minor compaction, let's see what files are applicable
    int start = 0;
    double ratio = comConf.getCompactionRatio();
    if (mayUseOffPeak) {
        ratio = comConf.getCompactionRatioOffPeak();
        LOG.info("Running an off-peak compaction, selection ratio = " + ratio);
    }
    // get store file sizes for incremental compacting selection.
    final int countOfFiles = candidates.size();
    long[] fileSizes = new long[countOfFiles];
    long[] sumSize = new long[countOfFiles];
    for (int i = countOfFiles - 1; i >= 0; --i) {
        StoreFile file = candidates.get(i);
        fileSizes[i] = file.getReader().length();
        // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
        int tooFar = i + comConf.getMaxFilesToCompact() - 1;
        sumSize[i] = fileSizes[i] + ((i + 1 < countOfFiles) ? sumSize[i + 1] : 0) - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
    }
    while (countOfFiles - start >= comConf.getMinFilesToCompact() && fileSizes[start] > Math.max(comConf.getMinCompactSize(), (long) (sumSize[start + 1] * ratio))) {
        ++start;
    }
    if (start < countOfFiles) {
        LOG.info("Default compaction algorithm has selected " + (countOfFiles - start) + " files from " + countOfFiles + " candidates");
    } else if (mayBeStuck) {
        // We may be stuck. Compact the latest files if we can.
        int filesToLeave = candidates.size() - comConf.getMinFilesToCompact();
        if (filesToLeave >= 0) {
            start = filesToLeave;
        }
    }
    candidates.subList(0, start).clear();
    return candidates;
}