Examples with StoreFile org.apache.hadoop.hbase.regionserver.StoreFile used on opensource projects

Example 31 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class SortedCompactionPolicy method getCurrentEligibleFiles.

protected ArrayList<StoreFile> getCurrentEligibleFiles(ArrayList<StoreFile> candidateFiles, final List<StoreFile> filesCompacting) {
    // candidates = all storefiles not already in compaction queue
    if (!filesCompacting.isEmpty()) {
        // exclude all files older than the newest file we're currently
        // compacting. this allows us to preserve contiguity (HBASE-2856)
        StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
        int idx = candidateFiles.indexOf(last);
        Preconditions.checkArgument(idx != -1);
        candidateFiles.subList(0, idx + 1).clear();
    }
    return candidateFiles;
}

Example 32 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class StripeCompactionPolicy method selectSingleStripeCompaction.

protected StripeCompactionRequest selectSingleStripeCompaction(StripeInformationProvider si, boolean includeL0, boolean canDropDeletesWithoutL0, boolean isOffpeak) throws IOException {
    ArrayList<ImmutableList<StoreFile>> stripes = si.getStripes();
    int bqIndex = -1;
    List<StoreFile> bqSelection = null;
    int stripeCount = stripes.size();
    long bqTotalSize = -1;
    for (int i = 0; i < stripeCount; ++i) {
        // If we want to compact L0 to drop deletes, we only want whole-stripe compactions.
        // So, pass includeL0 as 2nd parameter to indicate that.
        List<StoreFile> selection = selectSimpleCompaction(stripes.get(i), !canDropDeletesWithoutL0 && includeL0, isOffpeak);
        if (selection.isEmpty())
            continue;
        long size = 0;
        for (StoreFile sf : selection) {
            size += sf.getReader().length();
        }
        if (bqSelection == null || selection.size() > bqSelection.size() || (selection.size() == bqSelection.size() && size < bqTotalSize)) {
            bqSelection = selection;
            bqIndex = i;
            bqTotalSize = size;
        }
    }
    if (bqSelection == null) {
        LOG.debug("No good compaction is possible in any stripe");
        return null;
    }
    List<StoreFile> filesToCompact = new ArrayList<>(bqSelection);
    // See if we can, and need to, split this stripe.
    int targetCount = 1;
    long targetKvs = Long.MAX_VALUE;
    boolean hasAllFiles = filesToCompact.size() == stripes.get(bqIndex).size();
    String splitString = "";
    if (hasAllFiles && bqTotalSize >= config.getSplitSize()) {
        if (includeL0) {
            // So, if we might split, don't compact the stripe with L0.
            return null;
        }
        Pair<Long, Integer> kvsAndCount = estimateTargetKvs(filesToCompact, config.getSplitCount());
        targetKvs = kvsAndCount.getFirst();
        targetCount = kvsAndCount.getSecond();
        splitString = "; the stripe will be split into at most " + targetCount + " stripes with " + targetKvs + " target KVs";
    }
    LOG.debug("Found compaction in a stripe with end key [" + Bytes.toString(si.getEndRow(bqIndex)) + "], with " + filesToCompact.size() + " files of total size " + bqTotalSize + splitString);
    // See if we can drop deletes.
    StripeCompactionRequest req;
    if (includeL0) {
        assert hasAllFiles;
        List<StoreFile> l0Files = si.getLevel0Files();
        LOG.debug("Adding " + l0Files.size() + " files to compaction to be able to drop deletes");
        ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<>();
        sfs.addSublist(filesToCompact);
        sfs.addSublist(l0Files);
        req = new BoundaryStripeCompactionRequest(sfs, si.getStripeBoundaries());
    } else {
        req = new SplitStripeCompactionRequest(filesToCompact, si.getStartRow(bqIndex), si.getEndRow(bqIndex), targetCount, targetKvs);
    }
    if (hasAllFiles && (canDropDeletesWithoutL0 || includeL0)) {
        req.setMajorRange(si.getStartRow(bqIndex), si.getEndRow(bqIndex));
    }
    req.getRequest().setOffPeak(isOffpeak);
    return req;
}

Example 33 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class CompactionRequest method recalculateSize.

/**
   * Recalculate the size of the compaction based on current files.
   * @param files files that should be included in the compaction
   */
private void recalculateSize() {
    long sz = 0;
    for (StoreFile sf : this.filesToCompact) {
        StoreFileReader r = sf.getReader();
        sz += r == null ? 0 : r.length();
    }
    this.totalSize = sz;
}

Example 34 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class Compactor method compact.

protected List<Path> compact(final CompactionRequest request, InternalScannerFactory scannerFactory, CellSinkFactory<T> sinkFactory, ThroughputController throughputController, User user) throws IOException {
    FileDetails fd = getFileDetails(request.getFiles(), request.isAllFiles());
    this.progress = new CompactionProgress(fd.maxKeyCount);
    // Find the smallest read point across all the Scanners.
    long smallestReadPoint = getSmallestReadPoint();
    List<StoreFileScanner> scanners;
    Collection<StoreFile> readersToClose;
    T writer = null;
    boolean dropCache;
    if (request.isMajor() || request.isAllFiles()) {
        dropCache = this.dropCacheMajor;
    } else {
        dropCache = this.dropCacheMinor;
    }
    if (this.conf.getBoolean("hbase.regionserver.compaction.private.readers", true)) {
        // clone all StoreFiles, so we'll do the compaction on a independent copy of StoreFiles,
        // HFiles, and their readers
        readersToClose = new ArrayList<>(request.getFiles().size());
        for (StoreFile f : request.getFiles()) {
            StoreFile clonedStoreFile = f.cloneForReader();
            // create the reader after the store file is cloned in case
            // the sequence id is used for sorting in scanners
            clonedStoreFile.createReader();
            readersToClose.add(clonedStoreFile);
        }
        scanners = createFileScanners(readersToClose, smallestReadPoint, dropCache);
    } else {
        readersToClose = Collections.emptyList();
        scanners = createFileScanners(request.getFiles(), smallestReadPoint, dropCache);
    }
    InternalScanner scanner = null;
    boolean finished = false;
    try {
        /* Include deletes, unless we are doing a major compaction */
        ScanType scanType = scannerFactory.getScanType(request);
        scanner = preCreateCoprocScanner(request, scanType, fd.earliestPutTs, scanners, user, smallestReadPoint);
        if (scanner == null) {
            scanner = scannerFactory.createScanner(scanners, scanType, fd, smallestReadPoint);
        }
        scanner = postCreateCoprocScanner(request, scanType, scanner, user);
        if (scanner == null) {
            // NULL scanner returned from coprocessor hooks means skip normal processing.
            return new ArrayList<>();
        }
        boolean cleanSeqId = false;
        if (fd.minSeqIdToKeep > 0) {
            smallestReadPoint = Math.min(fd.minSeqIdToKeep, smallestReadPoint);
            cleanSeqId = true;
        }
        writer = sinkFactory.createWriter(scanner, fd, dropCache);
        finished = performCompaction(fd, scanner, writer, smallestReadPoint, cleanSeqId, throughputController, request.isAllFiles(), request.getFiles().size());
        if (!finished) {
            throw new InterruptedIOException("Aborting compaction of store " + store + " in region " + store.getRegionInfo().getRegionNameAsString() + " because it was interrupted.");
        }
    } finally {
        Closeables.close(scanner, true);
        for (StoreFile f : readersToClose) {
            try {
                f.closeReader(true);
            } catch (IOException e) {
                LOG.warn("Exception closing " + f, e);
            }
        }
        if (!finished && writer != null) {
            abortWriter(writer);
        }
    }
    assert finished : "We should have exited the method on all error paths";
    assert writer != null : "Writer should be non-null if no error";
    return commitWriter(writer, fd, request);
}

Example 35 with StoreFile

use of org.apache.hadoop.hbase.regionserver.StoreFile in project hbase by apache.

the class HFileArchiver method deleteStoreFilesWithoutArchiving.

/**
   * Just do a simple delete of the given store files
   * <p>
   * A best effort is made to delete each of the files, rather than bailing on the first failure.
   * <p>
   * This method is preferable to {@link #deleteFilesWithoutArchiving(Collection)} since it consumes
   * less resources, but is limited in terms of usefulness
   * @param compactedFiles store files to delete from the file system.
   * @throws IOException if a file cannot be deleted. All files will be attempted to deleted before
   *           throwing the exception, rather than failing at the first file.
   */
private static void deleteStoreFilesWithoutArchiving(Collection<StoreFile> compactedFiles) throws IOException {
    LOG.debug("Deleting store files without archiving.");
    List<IOException> errors = new ArrayList<>(0);
    for (StoreFile hsf : compactedFiles) {
        try {
            hsf.deleteReader();
        } catch (IOException e) {
            LOG.error("Failed to delete store file:" + hsf.getPath());
            errors.add(e);
        }
    }
    if (errors.size() > 0) {
        throw MultipleIOException.createIOException(errors);
    }
}