Examples with CompactionStats org.apache.accumulo.server.compaction.CompactionStats used on opensource projects

Example 1 with CompactionStats

use of org.apache.accumulo.server.compaction.CompactionStats in project accumulo by apache.

the class MinorCompactor method call.

@Override
public CompactionStats call() {
    final String outputFileName = getOutputFile();
    log.trace("Begin minor compaction {} {}", outputFileName, getExtent());
    // output to new MapFile with a temporary name
    int sleepTime = 100;
    double growthFactor = 4;
    // 3 minutes
    int maxSleepTime = 1000 * 60 * 3;
    boolean reportedProblem = false;
    int retryCounter = 0;
    runningCompactions.add(this);
    try {
        do {
            try {
                CompactionStats ret = super.call();
                if (reportedProblem) {
                    ProblemReports.getInstance(tabletServer.getContext()).deleteProblemReport(getExtent().tableId(), ProblemType.FILE_WRITE, outputFileName);
                }
                return ret;
            } catch (IOException | UnsatisfiedLinkError e) {
                log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName);
                ProblemReports.getInstance(tabletServer.getContext()).report(new ProblemReport(getExtent().tableId(), ProblemType.FILE_WRITE, outputFileName, e));
                reportedProblem = true;
            } catch (RuntimeException | NoClassDefFoundError e) {
                // if this is coming from a user iterator, it is possible that the user could change the
                // iterator config and that the minor compaction would succeed
                // If the minor compaction stalls for too long during recovery, it can interfere with
                // other tables loading
                // Throw exception if this happens so assignments can be rescheduled.
                ProblemReports.getInstance(tabletServer.getContext()).report(new ProblemReport(getExtent().tableId(), ProblemType.FILE_WRITE, outputFileName, e));
                if (retryCounter >= 4 && mincReason.equals(MinorCompactionReason.RECOVERY)) {
                    log.warn("MinC ({}) is stuck for too long during recovery, throwing error to reschedule.", getExtent(), e);
                    throw new RuntimeException(e);
                }
                log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName, e);
                reportedProblem = true;
                retryCounter++;
            } catch (CompactionCanceledException e) {
                throw new IllegalStateException(e);
            }
            int sleep = sleepTime + random.nextInt(sleepTime);
            log.debug("MinC failed sleeping {} ms before retrying", sleep);
            sleepUninterruptibly(sleep, TimeUnit.MILLISECONDS);
            sleepTime = (int) Math.round(Math.min(maxSleepTime, sleepTime * growthFactor));
            // clean up
            try {
                if (getVolumeManager().exists(new Path(outputFileName))) {
                    getVolumeManager().deleteRecursively(new Path(outputFileName));
                }
            } catch (IOException e) {
                log.warn("Failed to delete failed MinC file {} {}", outputFileName, e.getMessage());
            }
            if (isTableDeleting())
                return new CompactionStats(0, 0);
        } while (true);
    } finally {
        thread = null;
        runningCompactions.remove(this);
    }
}

Example 2 with CompactionStats

use of org.apache.accumulo.server.compaction.CompactionStats in project accumulo by apache.

the class CompactableImpl method compact.

@Override
public void compact(CompactionServiceId service, CompactionJob job, RateLimiter readLimiter, RateLimiter writeLimiter, long queuedTime) {
    Optional<CompactionInfo> ocInfo = reserveFilesForCompaction(service, job);
    if (ocInfo.isEmpty())
        return;
    var cInfo = ocInfo.get();
    StoredTabletFile newFile = null;
    long startTime = System.currentTimeMillis();
    CompactionKind kind = job.getKind();
    CompactionStats stats = new CompactionStats();
    try {
        TabletLogger.compacting(getExtent(), job, cInfo.localCompactionCfg);
        tablet.incrementStatusMajor();
        var check = new CompactionCheck(service, kind, cInfo.checkCompactionId);
        TabletFile tmpFileName = tablet.getNextMapFilenameForMajc(cInfo.propagateDeletes);
        var compactEnv = new MajCEnv(kind, check, readLimiter, writeLimiter, cInfo.propagateDeletes);
        SortedMap<StoredTabletFile, DataFileValue> allFiles = tablet.getDatafiles();
        HashMap<StoredTabletFile, DataFileValue> compactFiles = new HashMap<>();
        cInfo.jobFiles.forEach(file -> compactFiles.put(file, allFiles.get(file)));
        stats = CompactableUtils.compact(tablet, job, cInfo, compactEnv, compactFiles, tmpFileName);
        newFile = CompactableUtils.bringOnline(tablet.getDatafileManager(), cInfo, stats, compactFiles, allFiles, kind, tmpFileName);
        TabletLogger.compacted(getExtent(), job, newFile);
    } catch (CompactionCanceledException cce) {
        log.debug("Compaction canceled {} ", getExtent());
    } catch (Exception e) {
        newFile = null;
        throw new RuntimeException(e);
    } finally {
        completeCompaction(job, cInfo.jobFiles, newFile);
        tablet.updateTimer(MAJOR, queuedTime, startTime, stats.getEntriesRead(), newFile == null);
    }
}

Example 3 with CompactionStats

use of org.apache.accumulo.server.compaction.CompactionStats in project accumulo by apache.

the class Tablet method minorCompact.

DataFileValue minorCompact(InMemoryMap memTable, TabletFile tmpDatafile, TabletFile newDatafile, long queued, CommitSession commitSession, long flushId, MinorCompactionReason mincReason) {
    boolean failed = false;
    long start = System.currentTimeMillis();
    timer.incrementStatusMinor();
    long count = 0;
    String oldName = Thread.currentThread().getName();
    try {
        Thread.currentThread().setName("Minor compacting " + this.extent);
        CompactionStats stats;
        Span span = TraceUtil.startSpan(this.getClass(), "minorCompact::write");
        try (Scope scope = span.makeCurrent()) {
            count = memTable.getNumEntries();
            MinorCompactor compactor = new MinorCompactor(tabletServer, this, memTable, tmpDatafile, mincReason, tableConfiguration);
            stats = compactor.call();
        } catch (Exception e) {
            TraceUtil.setException(span, e, true);
            throw e;
        } finally {
            span.end();
        }
        Span span2 = TraceUtil.startSpan(this.getClass(), "minorCompact::bringOnline");
        try (Scope scope = span2.makeCurrent()) {
            var storedFile = getDatafileManager().bringMinorCompactionOnline(tmpDatafile, newDatafile, new DataFileValue(stats.getFileSize(), stats.getEntriesWritten()), commitSession, flushId);
            storedFile.ifPresent(stf -> compactable.filesAdded(true, List.of(stf)));
        } catch (Exception e) {
            TraceUtil.setException(span2, e, true);
            throw e;
        } finally {
            span2.end();
        }
        return new DataFileValue(stats.getFileSize(), stats.getEntriesWritten());
    } catch (Exception | Error e) {
        failed = true;
        throw new RuntimeException("Exception occurred during minor compaction on " + extent, e);
    } finally {
        Thread.currentThread().setName(oldName);
        try {
            getTabletMemory().finalizeMinC();
        } catch (Exception t) {
            log.error("Failed to free tablet memory on {}", extent, t);
        }
        if (!failed) {
            lastMinorCompactionFinishTime = System.currentTimeMillis();
        }
        TabletServerMinCMetrics minCMetrics = getTabletServer().getMinCMetrics();
        minCMetrics.addActive(lastMinorCompactionFinishTime - start);
        timer.updateTime(Operation.MINOR, queued, start, count, failed);
        minCMetrics.addQueued(start - queued);
    }
}