Examples with CloseChecker org.apache.hadoop.hbase.regionserver.compactions.CloseChecker used on opensource projects

Example 1 with CloseChecker

use of org.apache.hadoop.hbase.regionserver.compactions.CloseChecker in project hbase by apache.

the class DefaultMobStoreCompactor method performCompaction.

/**
 * Performs compaction on a column family with the mob flag enabled. This works only when MOB
 * compaction is explicitly requested (by User), or by Master There are two modes of a MOB
 * compaction:<br>
 * <p>
 * <ul>
 * <li>1. Full mode - when all MOB data for a region is compacted into a single MOB file.
 * <li>2. I/O optimized mode - for use cases with no or infrequent updates/deletes of a <br>
 * MOB data. The main idea behind i/o optimized compaction is to limit maximum size of a MOB file
 * produced during compaction and to limit I/O write/read amplification.
 * </ul>
 * The basic algorithm of compaction is the following: <br>
 * 1. If the Put cell has a mob reference tag, the cell's value is the path of the mob file.
 * <ol>
 * <li>If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
 * directly copy the (with mob tag) cell into the new store file.</li>
 * <li>Otherwise, retrieve the mob cell from the mob file, and writes a copy of the cell into the
 * new store file.</li>
 * </ol>
 * 2. If the Put cell doesn't have a reference tag.
 * <ol>
 * <li>If the value size of a cell is larger than the threshold, this cell is regarded as a mob,
 * write this cell to a mob file, and write the path of this mob file to the store file.</li>
 * <li>Otherwise, directly write this cell into the store file.</li>
 * </ol>
 * @param fd File details
 * @param scanner Where to read from.
 * @param smallestReadPoint Smallest read point.
 * @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
 * @param throughputController The compaction throughput controller.
 * @param major Is a major compaction.
 * @param numofFilesToCompact the number of files to compact
 * @return Whether compaction ended; false if it was interrupted for any reason.
 */
@Override
protected boolean performCompaction(FileDetails fd, InternalScanner scanner, long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController, boolean major, int numofFilesToCompact) throws IOException {
    long bytesWrittenProgressForLog = 0;
    long bytesWrittenProgressForShippedCall = 0;
    // Clear old mob references
    mobRefSet.get().clear();
    boolean isUserRequest = userRequest.get();
    boolean compactMOBs = major && isUserRequest;
    boolean discardMobMiss = conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY, MobConstants.DEFAULT_MOB_DISCARD_MISS);
    if (discardMobMiss) {
        LOG.warn("{}=true. This is unsafe setting recommended only when first upgrading to a version" + " with the distributed mob compaction feature on a cluster that has experienced MOB data " + "corruption.", MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY);
    }
    long maxMobFileSize = conf.getLong(MobConstants.MOB_COMPACTION_MAX_FILE_SIZE_KEY, MobConstants.DEFAULT_MOB_COMPACTION_MAX_FILE_SIZE);
    boolean ioOptimizedMode = this.ioOptimizedMode && !disableIO.get();
    LOG.info("Compact MOB={} optimized configured={} optimized enabled={} maximum MOB file size={}" + " major={} store={}", compactMOBs, this.ioOptimizedMode, ioOptimizedMode, maxMobFileSize, major, getStoreInfo());
    // Since scanner.next() can return 'false' but still be delivering data,
    // we have to use a do/while loop.
    List<Cell> cells = new ArrayList<>();
    // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
    long currentTime = EnvironmentEdgeManager.currentTime();
    long lastMillis = 0;
    if (LOG.isDebugEnabled()) {
        lastMillis = currentTime;
    }
    CloseChecker closeChecker = new CloseChecker(conf, currentTime);
    String compactionName = ThroughputControlUtil.getNameForThrottling(store, "compaction");
    long now = 0;
    boolean hasMore;
    byte[] fileName = null;
    StoreFileWriter mobFileWriter = null;
    /*
     * mobCells are used only to decide if we need to commit or abort current MOB output file.
     */
    long mobCells = 0;
    long cellsCountCompactedToMob = 0, cellsCountCompactedFromMob = 0;
    long cellsSizeCompactedToMob = 0, cellsSizeCompactedFromMob = 0;
    boolean finished = false;
    ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
    throughputController.start(compactionName);
    KeyValueScanner kvs = (scanner instanceof KeyValueScanner) ? (KeyValueScanner) scanner : null;
    long shippedCallSizeLimit = (long) numofFilesToCompact * this.store.getColumnFamilyDescriptor().getBlocksize();
    Cell mobCell = null;
    try {
        mobFileWriter = newMobWriter(fd, major);
        fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
        do {
            hasMore = scanner.next(cells, scannerContext);
            currentTime = EnvironmentEdgeManager.currentTime();
            if (LOG.isDebugEnabled()) {
                now = currentTime;
            }
            if (closeChecker.isTimeLimit(store, currentTime)) {
                progress.cancel();
                return false;
            }
            for (Cell c : cells) {
                if (compactMOBs) {
                    if (MobUtils.isMobReferenceCell(c)) {
                        String fName = MobUtils.getMobFileName(c);
                        // Added to support migration
                        try {
                            mobCell = mobStore.resolve(c, true, false).getCell();
                        } catch (DoNotRetryIOException e) {
                            if (discardMobMiss && e.getCause() != null && e.getCause() instanceof FileNotFoundException) {
                                LOG.error("Missing MOB cell: file={} not found cell={}", fName, c);
                                continue;
                            } else {
                                throw e;
                            }
                        }
                        if (discardMobMiss && mobCell.getValueLength() == 0) {
                            LOG.error("Missing MOB cell value: file={} mob cell={} cell={}", fName, mobCell, c);
                            continue;
                        } else if (mobCell.getValueLength() == 0) {
                            String errMsg = String.format("Found 0 length MOB cell in a file=%s mob cell=%s " + " cell=%s", fName, mobCell, c);
                            throw new IOException(errMsg);
                        }
                        if (mobCell.getValueLength() > mobSizeThreshold) {
                            // put the mob data back to the MOB store file
                            PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                            if (!ioOptimizedMode) {
                                mobFileWriter.append(mobCell);
                                mobCells++;
                                writer.append(MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
                            } else {
                                // I/O optimized mode
                                // Check if MOB cell origin file size is
                                // greater than threshold
                                Long size = mobLengthMap.get().get(fName);
                                if (size == null) {
                                    // FATAL error (we should never get here though), abort compaction
                                    // This error means that meta section of store file does not contain
                                    // MOB file, which has references in at least one cell from this store file
                                    String msg = String.format("Found an unexpected MOB file during compaction %s, aborting compaction %s", fName, getStoreInfo());
                                    throw new IOException(msg);
                                }
                                // Can not be null
                                if (size < maxMobFileSize) {
                                    // If MOB cell origin file is below threshold
                                    // it is get compacted
                                    mobFileWriter.append(mobCell);
                                    // Update number of mobCells in a current mob writer
                                    mobCells++;
                                    writer.append(MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
                                    // Update total size of the output (we do not take into account
                                    // file compression yet)
                                    long len = mobFileWriter.getPos();
                                    if (len > maxMobFileSize) {
                                        LOG.debug("Closing output MOB File, length={} file={}, store={}", len, mobFileWriter.getPath().getName(), getStoreInfo());
                                        commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
                                        mobFileWriter = newMobWriter(fd, major);
                                        fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
                                        mobCells = 0;
                                    }
                                } else {
                                    // We leave large MOB file as is (is not compacted),
                                    // then we update set of MOB file references
                                    // and append mob cell directly to the store's writer
                                    Optional<TableName> refTable = MobUtils.getTableName(c);
                                    if (refTable.isPresent()) {
                                        mobRefSet.get().put(refTable.get(), fName);
                                        writer.append(c);
                                    } else {
                                        throw new IOException(String.format("MOB cell did not contain a tablename " + "tag. should not be possible. see ref guide on mob troubleshooting. " + "store=%s cell=%s", getStoreInfo(), c));
                                    }
                                }
                            }
                        } else {
                            // If MOB value is less than threshold, append it directly to a store file
                            PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                            writer.append(mobCell);
                            cellsCountCompactedFromMob++;
                            cellsSizeCompactedFromMob += mobCell.getValueLength();
                        }
                    } else {
                        // Not a MOB reference cell
                        int size = c.getValueLength();
                        if (size > mobSizeThreshold) {
                            // This MOB cell comes from a regular store file
                            // therefore we store it into original mob output
                            mobFileWriter.append(c);
                            writer.append(MobUtils.createMobRefCell(c, fileName, this.mobStore.getRefCellTags()));
                            mobCells++;
                            cellsCountCompactedToMob++;
                            cellsSizeCompactedToMob += c.getValueLength();
                            if (ioOptimizedMode) {
                                // Update total size of the output (we do not take into account
                                // file compression yet)
                                long len = mobFileWriter.getPos();
                                if (len > maxMobFileSize) {
                                    commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
                                    mobFileWriter = newMobWriter(fd, major);
                                    fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
                                    mobCells = 0;
                                }
                            }
                        } else {
                            // Not a MOB cell, write it directly to a store file
                            writer.append(c);
                        }
                    }
                } else if (c.getTypeByte() != KeyValue.Type.Put.getCode()) {
                    // Not a major compaction or major with MOB disabled
                    // If the kv type is not put, directly write the cell
                    // to the store file.
                    writer.append(c);
                } else if (MobUtils.isMobReferenceCell(c)) {
                    // Not a major MOB compaction, Put MOB reference
                    if (MobUtils.hasValidMobRefCellValue(c)) {
                        // We do not check mobSizeThreshold during normal compaction,
                        // leaving it to a MOB compaction run
                        Optional<TableName> refTable = MobUtils.getTableName(c);
                        if (refTable.isPresent()) {
                            mobRefSet.get().put(refTable.get(), MobUtils.getMobFileName(c));
                            writer.append(c);
                        } else {
                            throw new IOException(String.format("MOB cell did not contain a tablename " + "tag. should not be possible. see ref guide on mob troubleshooting. " + "store=%s cell=%s", getStoreInfo(), c));
                        }
                    } else {
                        String errMsg = String.format("Corrupted MOB reference: %s", c.toString());
                        throw new IOException(errMsg);
                    }
                } else if (c.getValueLength() <= mobSizeThreshold) {
                    // If the value size of a cell is not larger than the threshold, directly write it to
                    // the store file.
                    writer.append(c);
                } else {
                    // If the value size of a cell is larger than the threshold, it's regarded as a mob,
                    // write this cell to a mob file, and write the path to the store file.
                    mobCells++;
                    // append the original keyValue in the mob file.
                    mobFileWriter.append(c);
                    Cell reference = MobUtils.createMobRefCell(c, fileName, this.mobStore.getRefCellTags());
                    // write the cell whose value is the path of a mob file to the store file.
                    writer.append(reference);
                    cellsCountCompactedToMob++;
                    cellsSizeCompactedToMob += c.getValueLength();
                    if (ioOptimizedMode) {
                        long len = mobFileWriter.getPos();
                        if (len > maxMobFileSize) {
                            commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
                            mobFileWriter = newMobWriter(fd, major);
                            fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
                            mobCells = 0;
                        }
                    }
                }
                int len = c.getSerializedSize();
                ++progress.currentCompactedKVs;
                progress.totalCompactedSize += len;
                bytesWrittenProgressForShippedCall += len;
                if (LOG.isDebugEnabled()) {
                    bytesWrittenProgressForLog += len;
                }
                throughputController.control(compactionName, len);
                if (closeChecker.isSizeLimit(store, len)) {
                    progress.cancel();
                    return false;
                }
                if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) {
                    ((ShipperListener) writer).beforeShipped();
                    kvs.shipped();
                    bytesWrittenProgressForShippedCall = 0;
                }
            }
            // logging at DEBUG level
            if (LOG.isDebugEnabled()) {
                if ((now - lastMillis) >= COMPACTION_PROGRESS_LOG_INTERVAL) {
                    String rate = String.format("%.2f", (bytesWrittenProgressForLog / 1024.0) / ((now - lastMillis) / 1000.0));
                    LOG.debug("Compaction progress: {} {}, rate={} KB/sec, throughputController is {}", compactionName, progress, rate, throughputController);
                    lastMillis = now;
                    bytesWrittenProgressForLog = 0;
                }
            }
            cells.clear();
        } while (hasMore);
        finished = true;
    } catch (InterruptedException e) {
        progress.cancel();
        throw new InterruptedIOException("Interrupted while control throughput of compacting " + compactionName);
    } catch (IOException t) {
        String msg = "Mob compaction failed for region: " + store.getRegionInfo().getEncodedName();
        throw new IOException(msg, t);
    } finally {
        // Clone last cell in the final because writer will append last cell when committing. If
        // don't clone here and once the scanner get closed, then the memory of last cell will be
        // released. (HBASE-22582)
        ((ShipperListener) writer).beforeShipped();
        throughputController.finish(compactionName);
        if (!finished && mobFileWriter != null) {
            // Remove all MOB references because compaction failed
            clearThreadLocals();
            // Abort writer
            LOG.debug("Aborting writer for {} because of a compaction failure, Store {}", mobFileWriter.getPath(), getStoreInfo());
            abortWriter(mobFileWriter);
        }
    }
    // Commit last MOB writer
    commitOrAbortMobWriter(mobFileWriter, fd.maxSeqId, mobCells, major);
    mobStore.updateCellsCountCompactedFromMob(cellsCountCompactedFromMob);
    mobStore.updateCellsCountCompactedToMob(cellsCountCompactedToMob);
    mobStore.updateCellsSizeCompactedFromMob(cellsSizeCompactedFromMob);
    mobStore.updateCellsSizeCompactedToMob(cellsSizeCompactedToMob);
    progress.complete();
    return true;
}

Example 2 with CloseChecker

use of org.apache.hadoop.hbase.regionserver.compactions.CloseChecker in project hbase by apache.

the class FaultyMobStoreCompactor method performCompaction.

@Override
protected boolean performCompaction(FileDetails fd, InternalScanner scanner, long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController, boolean major, int numofFilesToCompact) throws IOException {
    totalCompactions.incrementAndGet();
    if (major) {
        totalMajorCompactions.incrementAndGet();
    }
    long bytesWrittenProgressForLog = 0;
    long bytesWrittenProgressForShippedCall = 0;
    // Clear old mob references
    mobRefSet.get().clear();
    boolean isUserRequest = userRequest.get();
    boolean compactMOBs = major && isUserRequest;
    boolean discardMobMiss = conf.getBoolean(MobConstants.MOB_UNSAFE_DISCARD_MISS_KEY, MobConstants.DEFAULT_MOB_DISCARD_MISS);
    boolean mustFail = false;
    if (compactMOBs) {
        mobCounter.incrementAndGet();
        double dv = rnd.nextDouble();
        if (dv < failureProb) {
            mustFail = true;
            totalFailures.incrementAndGet();
        }
    }
    FileSystem fs = store.getFileSystem();
    // Since scanner.next() can return 'false' but still be delivering data,
    // we have to use a do/while loop.
    List<Cell> cells = new ArrayList<>();
    // Limit to "hbase.hstore.compaction.kv.max" (default 10) to avoid OOME
    long currentTime = EnvironmentEdgeManager.currentTime();
    long lastMillis = 0;
    if (LOG.isDebugEnabled()) {
        lastMillis = currentTime;
    }
    CloseChecker closeChecker = new CloseChecker(conf, currentTime);
    String compactionName = ThroughputControlUtil.getNameForThrottling(store, "compaction");
    long now = 0;
    boolean hasMore;
    Path path = MobUtils.getMobFamilyPath(conf, store.getTableName(), store.getColumnFamilyName());
    byte[] fileName = null;
    StoreFileWriter mobFileWriter = null;
    long mobCells = 0;
    long cellsCountCompactedToMob = 0, cellsCountCompactedFromMob = 0;
    long cellsSizeCompactedToMob = 0, cellsSizeCompactedFromMob = 0;
    boolean finished = false;
    ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(compactionKVMax).build();
    throughputController.start(compactionName);
    KeyValueScanner kvs = (scanner instanceof KeyValueScanner) ? (KeyValueScanner) scanner : null;
    long shippedCallSizeLimit = (long) numofFilesToCompact * this.store.getColumnFamilyDescriptor().getBlocksize();
    Cell mobCell = null;
    long counter = 0;
    long countFailAt = -1;
    if (mustFail) {
        // randomly fail fast
        countFailAt = rnd.nextInt(100);
    }
    try {
        try {
            mobFileWriter = mobStore.createWriterInTmp(new Date(fd.latestPutTs), fd.maxKeyCount, major ? majorCompactionCompression : minorCompactionCompression, store.getRegionInfo().getStartKey(), true);
            fileName = Bytes.toBytes(mobFileWriter.getPath().getName());
        } catch (IOException e) {
            // Bailing out
            LOG.error("Failed to create mob writer, ", e);
            throw e;
        }
        if (compactMOBs) {
            // Add the only reference we get for compact MOB case
            // because new store file will have only one MOB reference
            // in this case - of newly compacted MOB file
            mobRefSet.get().put(store.getTableName(), mobFileWriter.getPath().getName());
        }
        do {
            hasMore = scanner.next(cells, scannerContext);
            currentTime = EnvironmentEdgeManager.currentTime();
            if (LOG.isDebugEnabled()) {
                now = currentTime;
            }
            if (closeChecker.isTimeLimit(store, currentTime)) {
                progress.cancel();
                return false;
            }
            for (Cell c : cells) {
                counter++;
                if (compactMOBs) {
                    if (MobUtils.isMobReferenceCell(c)) {
                        if (counter == countFailAt) {
                            LOG.warn("INJECTED FAULT mobCounter={}", mobCounter.get());
                            throw new CorruptHFileException("injected fault");
                        }
                        String fName = MobUtils.getMobFileName(c);
                        // Added to support migration
                        try {
                            mobCell = mobStore.resolve(c, true, false).getCell();
                        } catch (DoNotRetryIOException e) {
                            if (discardMobMiss && e.getCause() != null && e.getCause() instanceof FileNotFoundException) {
                                LOG.error("Missing MOB cell: file={} not found cell={}", fName, c);
                                continue;
                            } else {
                                throw e;
                            }
                        }
                        if (discardMobMiss && mobCell.getValueLength() == 0) {
                            LOG.error("Missing MOB cell value: file={} cell={}", fName, mobCell);
                            continue;
                        }
                        if (mobCell.getValueLength() > mobSizeThreshold) {
                            // put the mob data back to the store file
                            PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                            mobFileWriter.append(mobCell);
                            writer.append(MobUtils.createMobRefCell(mobCell, fileName, this.mobStore.getRefCellTags()));
                            mobCells++;
                        } else {
                            // If MOB value is less than threshold, append it directly to a store file
                            PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                            writer.append(mobCell);
                            cellsCountCompactedFromMob++;
                            cellsSizeCompactedFromMob += mobCell.getValueLength();
                        }
                    } else {
                        // Not a MOB reference cell
                        int size = c.getValueLength();
                        if (size > mobSizeThreshold) {
                            mobFileWriter.append(c);
                            writer.append(MobUtils.createMobRefCell(c, fileName, this.mobStore.getRefCellTags()));
                            mobCells++;
                            cellsCountCompactedToMob++;
                            cellsSizeCompactedToMob += c.getValueLength();
                        } else {
                            writer.append(c);
                        }
                    }
                } else if (c.getTypeByte() != KeyValue.Type.Put.getCode()) {
                    // Not a major compaction or major with MOB disabled
                    // If the kv type is not put, directly write the cell
                    // to the store file.
                    writer.append(c);
                } else if (MobUtils.isMobReferenceCell(c)) {
                    // Not a major MOB compaction, Put MOB reference
                    if (MobUtils.hasValidMobRefCellValue(c)) {
                        int size = MobUtils.getMobValueLength(c);
                        if (size > mobSizeThreshold) {
                            // If the value size is larger than the threshold, it's regarded as a mob. Since
                            // its value is already in the mob file, directly write this cell to the store file
                            Optional<TableName> refTable = MobUtils.getTableName(c);
                            if (refTable.isPresent()) {
                                mobRefSet.get().put(refTable.get(), MobUtils.getMobFileName(c));
                                writer.append(c);
                            } else {
                                throw new IOException(String.format("MOB cell did not contain a tablename " + "tag. should not be possible. see ref guide on mob troubleshooting. " + "store=%s cell=%s", getStoreInfo(), c));
                            }
                        } else {
                            // If the value is not larger than the threshold, it's not regarded a mob. Retrieve
                            // the mob cell from the mob file, and write it back to the store file.
                            mobCell = mobStore.resolve(c, true, false).getCell();
                            if (mobCell.getValueLength() != 0) {
                                // put the mob data back to the store file
                                PrivateCellUtil.setSequenceId(mobCell, c.getSequenceId());
                                writer.append(mobCell);
                                cellsCountCompactedFromMob++;
                                cellsSizeCompactedFromMob += mobCell.getValueLength();
                            } else {
                                // If the value of a file is empty, there might be issues when retrieving,
                                // directly write the cell to the store file, and leave it to be handled by the
                                // next compaction.
                                LOG.error("Empty value for: " + c);
                                Optional<TableName> refTable = MobUtils.getTableName(c);
                                if (refTable.isPresent()) {
                                    mobRefSet.get().put(refTable.get(), MobUtils.getMobFileName(c));
                                    writer.append(c);
                                } else {
                                    throw new IOException(String.format("MOB cell did not contain a tablename " + "tag. should not be possible. see ref guide on mob troubleshooting. " + "store=%s cell=%s", getStoreInfo(), c));
                                }
                            }
                        }
                    } else {
                        LOG.error("Corrupted MOB reference: {}", c);
                        writer.append(c);
                    }
                } else if (c.getValueLength() <= mobSizeThreshold) {
                    // If the value size of a cell is not larger than the threshold, directly write it to
                    // the store file.
                    writer.append(c);
                } else {
                    // If the value size of a cell is larger than the threshold, it's regarded as a mob,
                    // write this cell to a mob file, and write the path to the store file.
                    mobCells++;
                    // append the original keyValue in the mob file.
                    mobFileWriter.append(c);
                    Cell reference = MobUtils.createMobRefCell(c, fileName, this.mobStore.getRefCellTags());
                    // write the cell whose value is the path of a mob file to the store file.
                    writer.append(reference);
                    cellsCountCompactedToMob++;
                    cellsSizeCompactedToMob += c.getValueLength();
                    // Add ref we get for compact MOB case
                    mobRefSet.get().put(store.getTableName(), mobFileWriter.getPath().getName());
                }
                int len = c.getSerializedSize();
                ++progress.currentCompactedKVs;
                progress.totalCompactedSize += len;
                bytesWrittenProgressForShippedCall += len;
                if (LOG.isDebugEnabled()) {
                    bytesWrittenProgressForLog += len;
                }
                throughputController.control(compactionName, len);
                if (closeChecker.isSizeLimit(store, len)) {
                    progress.cancel();
                    return false;
                }
                if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) {
                    ((ShipperListener) writer).beforeShipped();
                    kvs.shipped();
                    bytesWrittenProgressForShippedCall = 0;
                }
            }
            // logging at DEBUG level
            if (LOG.isDebugEnabled()) {
                if ((now - lastMillis) >= COMPACTION_PROGRESS_LOG_INTERVAL) {
                    String rate = String.format("%.2f", (bytesWrittenProgressForLog / 1024.0) / ((now - lastMillis) / 1000.0));
                    LOG.debug("Compaction progress: {} {}, rate={} KB/sec, throughputController is {}", compactionName, progress, rate, throughputController);
                    lastMillis = now;
                    bytesWrittenProgressForLog = 0;
                }
            }
            cells.clear();
        } while (hasMore);
        finished = true;
    } catch (InterruptedException e) {
        progress.cancel();
        throw new InterruptedIOException("Interrupted while control throughput of compacting " + compactionName);
    } catch (FileNotFoundException e) {
        LOG.error("MOB Stress Test FAILED, region: " + store.getRegionInfo().getEncodedName(), e);
        System.exit(-1);
    } catch (IOException t) {
        LOG.error("Mob compaction failed for region: " + store.getRegionInfo().getEncodedName());
        throw t;
    } finally {
        // Clone last cell in the final because writer will append last cell when committing. If
        // don't clone here and once the scanner get closed, then the memory of last cell will be
        // released. (HBASE-22582)
        ((ShipperListener) writer).beforeShipped();
        throughputController.finish(compactionName);
        if (!finished && mobFileWriter != null) {
            // Remove all MOB references because compaction failed
            mobRefSet.get().clear();
            // Abort writer
            abortWriter(mobFileWriter);
        }
    }
    if (mobFileWriter != null) {
        if (mobCells > 0) {
            // If the mob file is not empty, commit it.
            mobFileWriter.appendMetadata(fd.maxSeqId, major, mobCells);
            mobFileWriter.close();
            mobStore.commitFile(mobFileWriter.getPath(), path);
        } else {
            // If the mob file is empty, delete it instead of committing.
            abortWriter(mobFileWriter);
        }
    }
    mobStore.updateCellsCountCompactedFromMob(cellsCountCompactedFromMob);
    mobStore.updateCellsCountCompactedToMob(cellsCountCompactedToMob);
    mobStore.updateCellsSizeCompactedFromMob(cellsSizeCompactedFromMob);
    mobStore.updateCellsSizeCompactedToMob(cellsSizeCompactedToMob);
    progress.complete();
    return true;
}