Examples with HStoreFile org.apache.hadoop.hbase.regionserver.HStoreFile used on opensource projects

Example 26 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class DateTieredCompactionPolicy method shouldPerformMajorCompaction.

@Override
public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact) throws IOException {
    long mcTime = getNextMajorCompactTime(filesToCompact);
    if (filesToCompact == null || mcTime == 0) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("filesToCompact: " + filesToCompact + " mcTime: " + mcTime);
        }
        return false;
    }
    // TODO: Use better method for determining stamp of last major (HBASE-2990)
    long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
    long now = EnvironmentEdgeManager.currentTime();
    if (lowTimestamp <= 0L || lowTimestamp >= (now - mcTime)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("lowTimestamp: " + lowTimestamp + " lowTimestamp: " + lowTimestamp + " now: " + now + " mcTime: " + mcTime);
        }
        return false;
    }
    long cfTTL = this.storeConfigInfo.getStoreFileTtl();
    HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
    List<Long> boundaries = getCompactBoundariesForMajor(filesToCompact, now);
    boolean[] filesInWindow = new boolean[boundaries.size()];
    for (HStoreFile file : filesToCompact) {
        OptionalLong minTimestamp = file.getMinimumTimestamp();
        long oldest = minTimestamp.isPresent() ? now - minTimestamp.getAsLong() : Long.MIN_VALUE;
        if (cfTTL != Long.MAX_VALUE && oldest >= cfTTL) {
            LOG.debug("Major compaction triggered on store " + this + "; for TTL maintenance");
            return true;
        }
        if (!file.isMajorCompactionResult() || file.isBulkLoadResult()) {
            LOG.debug("Major compaction triggered on store " + this + ", because there are new files and time since last major compaction " + (now - lowTimestamp) + "ms");
            return true;
        }
        int lowerWindowIndex = Collections.binarySearch(boundaries, minTimestamp.orElse(Long.MAX_VALUE));
        int upperWindowIndex = Collections.binarySearch(boundaries, file.getMaximumTimestamp().orElse(Long.MAX_VALUE));
        // Handle boundary conditions and negative values of binarySearch
        lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex;
        upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex;
        if (lowerWindowIndex != upperWindowIndex) {
            LOG.debug("Major compaction triggered on store " + this + "; because file " + file.getPath() + " has data with timestamps cross window boundaries");
            return true;
        } else if (filesInWindow[upperWindowIndex]) {
            LOG.debug("Major compaction triggered on store " + this + "; because there are more than one file in some windows");
            return true;
        } else {
            filesInWindow[upperWindowIndex] = true;
        }
        hdfsBlocksDistribution.add(file.getHDFSBlockDistribution());
    }
    float blockLocalityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(DNS.getHostname(comConf.conf, DNS.ServerType.REGIONSERVER));
    if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) {
        LOG.debug("Major compaction triggered on store " + this + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")");
        return true;
    }
    LOG.debug("Skipping major compaction of " + this + ", because the files are already major compacted");
    return false;
}

Example 27 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class FIFOCompactionPolicy method hasExpiredStores.

private boolean hasExpiredStores(Collection<HStoreFile> files) {
    long currentTime = EnvironmentEdgeManager.currentTime();
    for (HStoreFile sf : files) {
        if (isEmptyStoreFile(sf)) {
            return true;
        }
        // Check MIN_VERSIONS is in HStore removeUnneededFiles
        long maxTs = sf.getReader().getMaxTimestamp();
        long maxTtl = storeConfigInfo.getStoreFileTtl();
        if (maxTtl == Long.MAX_VALUE || (currentTime - maxTtl < maxTs)) {
            continue;
        } else {
            return true;
        }
    }
    return false;
}

Example 28 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class RatioBasedCompactionPolicy method applyCompactionPolicy.

/**
 * -- Default minor compaction selection algorithm:
 * choose CompactSelection from candidates --
 * First exclude bulk-load files if indicated in configuration.
 * Start at the oldest file and stop when you find the first file that
 * meets compaction criteria:
 * (1) a recently-flushed, small file (i.e. <= minCompactSize)
 * OR
 * (2) within the compactRatio of sum(newer_files)
 * Given normal skew, any newer files will also meet this criteria
 * <p/>
 * Additional Note:
 * If fileSizes.size() >> maxFilesToCompact, we will recurse on
 * compact().  Consider the oldest files first to avoid a
 * situation where we always compact [end-threshold,end).  Then, the
 * last file becomes an aggregate of the previous compactions.
 *
 * normal skew:
 *
 *         older ----> newer (increasing seqID)
 *     _
 *    | |   _
 *    | |  | |   _
 *  --|-|- |-|- |-|---_-------_-------  minCompactSize
 *    | |  | |  | |  | |  _  | |
 *    | |  | |  | |  | | | | | |
 *    | |  | |  | |  | | | | | |
 * @param candidates pre-filtrate
 * @return filtered subset
 */
protected ArrayList<HStoreFile> applyCompactionPolicy(ArrayList<HStoreFile> candidates, boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
    if (candidates.isEmpty()) {
        return candidates;
    }
    // we're doing a minor compaction, let's see what files are applicable
    int start = 0;
    double ratio = comConf.getCompactionRatio();
    if (mayUseOffPeak) {
        ratio = comConf.getCompactionRatioOffPeak();
        LOG.info("Running an off-peak compaction, selection ratio = " + ratio);
    }
    // get store file sizes for incremental compacting selection.
    final int countOfFiles = candidates.size();
    long[] fileSizes = new long[countOfFiles];
    long[] sumSize = new long[countOfFiles];
    for (int i = countOfFiles - 1; i >= 0; --i) {
        HStoreFile file = candidates.get(i);
        fileSizes[i] = file.getReader().length();
        // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
        int tooFar = i + comConf.getMaxFilesToCompact() - 1;
        sumSize[i] = fileSizes[i] + ((i + 1 < countOfFiles) ? sumSize[i + 1] : 0) - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
    }
    while (countOfFiles - start >= comConf.getMinFilesToCompact() && fileSizes[start] > Math.max(comConf.getMinCompactSize(), (long) (sumSize[start + 1] * ratio))) {
        ++start;
    }
    if (start < countOfFiles) {
        LOG.info("Default compaction algorithm has selected " + (countOfFiles - start) + " files from " + countOfFiles + " candidates");
    } else if (mayBeStuck) {
        // We may be stuck. Compact the latest files if we can.
        int filesToLeave = candidates.size() - comConf.getMinFilesToCompact();
        if (filesToLeave >= 0) {
            start = filesToLeave;
        }
    }
    candidates.subList(0, start).clear();
    return candidates;
}

Example 29 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class RatioBasedCompactionPolicy method shouldPerformMajorCompaction.

/*
   * @param filesToCompact Files to compact. Can be null.
   * @return True if we should run a major compaction.
   */
@Override
public boolean shouldPerformMajorCompaction(Collection<HStoreFile> filesToCompact) throws IOException {
    boolean result = false;
    long mcTime = getNextMajorCompactTime(filesToCompact);
    if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) {
        return result;
    }
    // TODO: Use better method for determining stamp of last major (HBASE-2990)
    long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
    long now = EnvironmentEdgeManager.currentTime();
    if (lowTimestamp > 0L && lowTimestamp < (now - mcTime)) {
        String regionInfo;
        if (this.storeConfigInfo != null && this.storeConfigInfo instanceof HStore) {
            regionInfo = ((HStore) this.storeConfigInfo).getRegionInfo().getRegionNameAsString();
        } else {
            regionInfo = this.toString();
        }
        // Major compaction time has elapsed.
        long cfTTL = HConstants.FOREVER;
        if (this.storeConfigInfo != null) {
            cfTTL = this.storeConfigInfo.getStoreFileTtl();
        }
        if (filesToCompact.size() == 1) {
            // Single file
            HStoreFile sf = filesToCompact.iterator().next();
            OptionalLong minTimestamp = sf.getMinimumTimestamp();
            long oldest = minTimestamp.isPresent() ? now - minTimestamp.getAsLong() : Long.MIN_VALUE;
            if (sf.isMajorCompactionResult() && (cfTTL == Long.MAX_VALUE || oldest < cfTTL)) {
                float blockLocalityIndex = sf.getHDFSBlockDistribution().getBlockLocalityIndex(DNS.getHostname(comConf.conf, DNS.ServerType.REGIONSERVER));
                if (blockLocalityIndex < comConf.getMinLocalityToForceCompact()) {
                    LOG.debug("Major compaction triggered on only store " + regionInfo + "; to make hdfs blocks local, current blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")");
                    result = true;
                } else {
                    LOG.debug("Skipping major compaction of " + regionInfo + " because one (major) compacted file only, oldestTime " + oldest + "ms is < TTL=" + cfTTL + " and blockLocalityIndex is " + blockLocalityIndex + " (min " + comConf.getMinLocalityToForceCompact() + ")");
                }
            } else if (cfTTL != HConstants.FOREVER && oldest > cfTTL) {
                LOG.debug("Major compaction triggered on store " + regionInfo + ", because keyvalues outdated; time since last major compaction " + (now - lowTimestamp) + "ms");
                result = true;
            }
        } else {
            LOG.debug("Major compaction triggered on store " + regionInfo + "; time since last major compaction " + (now - lowTimestamp) + "ms");
            result = true;
        }
    }
    return result;
}

Example 30 with HStoreFile

use of org.apache.hadoop.hbase.regionserver.HStoreFile in project hbase by apache.

the class SortedCompactionPolicy method getCurrentEligibleFiles.

protected ArrayList<HStoreFile> getCurrentEligibleFiles(ArrayList<HStoreFile> candidateFiles, final List<HStoreFile> filesCompacting) {
    // candidates = all storefiles not already in compaction queue
    if (!filesCompacting.isEmpty()) {
        // exclude all files older than the newest file we're currently
        // compacting. this allows us to preserve contiguity (HBASE-2856)
        HStoreFile last = filesCompacting.get(filesCompacting.size() - 1);
        int idx = candidateFiles.indexOf(last);
        Preconditions.checkArgument(idx != -1);
        candidateFiles.subList(0, idx + 1).clear();
    }
    return candidateFiles;
}