Examples with FileCompactor org.apache.accumulo.server.compaction.FileCompactor used on opensource projects

Example 1 with FileCompactor

use of org.apache.accumulo.server.compaction.FileCompactor in project accumulo by apache.

the class Compactor method createCompactionJob.

/**
 * Create compaction runnable
 *
 * @param job
 *          compaction job
 * @param totalInputEntries
 *          object to capture total entries
 * @param totalInputBytes
 *          object to capture input file size
 * @param started
 *          started latch
 * @param stopped
 *          stopped latch
 * @param err
 *          reference to error
 * @return Runnable compaction job
 */
protected Runnable createCompactionJob(final TExternalCompactionJob job, final LongAdder totalInputEntries, final LongAdder totalInputBytes, final CountDownLatch started, final CountDownLatch stopped, final AtomicReference<Throwable> err) {
    return new Runnable() {

        @Override
        public void run() {
            // Its only expected that a single compaction runs at a time. Multiple compactions running
            // at a time could cause odd behavior like out of order and unexpected thrift calls to the
            // coordinator. This is a sanity check to ensure the expectation is met. Should this check
            // ever fail, it means there is a bug elsewhere.
            Preconditions.checkState(compactionRunning.compareAndSet(false, true));
            try {
                LOG.info("Starting up compaction runnable for job: {}", job);
                TCompactionStatusUpdate update = new TCompactionStatusUpdate(TCompactionState.STARTED, "Compaction started", -1, -1, -1);
                updateCompactionState(job, update);
                final AccumuloConfiguration tConfig;
                var extent = KeyExtent.fromThrift(job.getExtent());
                if (!job.getOverrides().isEmpty()) {
                    tConfig = new ConfigurationCopy(getContext().getTableConfiguration(extent.tableId()));
                    job.getOverrides().forEach((k, v) -> ((ConfigurationCopy) tConfig).set(k, v));
                    LOG.debug("Overriding table properties with {}", job.getOverrides());
                } else {
                    tConfig = getContext().getTableConfiguration(extent.tableId());
                }
                final TabletFile outputFile = new TabletFile(new Path(job.getOutputFile()));
                final Map<StoredTabletFile, DataFileValue> files = new TreeMap<>();
                job.getFiles().forEach(f -> {
                    files.put(new StoredTabletFile(f.getMetadataFileEntry()), new DataFileValue(f.getSize(), f.getEntries(), f.getTimestamp()));
                    totalInputEntries.add(f.getEntries());
                    totalInputBytes.add(f.getSize());
                });
                final List<IteratorSetting> iters = new ArrayList<>();
                job.getIteratorSettings().getIterators().forEach(tis -> iters.add(SystemIteratorUtil.toIteratorSetting(tis)));
                ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, queueName);
                FileCompactor compactor = new FileCompactor(getContext(), extent, files, outputFile, job.isPropagateDeletes(), cenv, iters, tConfig);
                LOG.trace("Starting compactor");
                started.countDown();
                org.apache.accumulo.server.compaction.CompactionStats stat = compactor.call();
                TCompactionStats cs = new TCompactionStats();
                cs.setEntriesRead(stat.getEntriesRead());
                cs.setEntriesWritten(stat.getEntriesWritten());
                cs.setFileSize(stat.getFileSize());
                JOB_HOLDER.setStats(cs);
                LOG.info("Compaction completed successfully {} ", job.getExternalCompactionId());
                // Update state when completed
                TCompactionStatusUpdate update2 = new TCompactionStatusUpdate(TCompactionState.SUCCEEDED, "Compaction completed successfully", -1, -1, -1);
                updateCompactionState(job, update2);
            } catch (Exception e) {
                LOG.error("Compaction failed", e);
                err.set(e);
            } finally {
                stopped.countDown();
                Preconditions.checkState(compactionRunning.compareAndSet(true, false));
            }
        }
    };
}

Example 2 with FileCompactor

use of org.apache.accumulo.server.compaction.FileCompactor in project accumulo by apache.

the class CompactableUtils method compact.

/**
 * Create the FileCompactor and finally call compact. Returns the Major CompactionStats.
 */
static CompactionStats compact(Tablet tablet, CompactionJob job, CompactableImpl.CompactionInfo cInfo, CompactionEnv cenv, Map<StoredTabletFile, DataFileValue> compactFiles, TabletFile tmpFileName) throws IOException, CompactionCanceledException {
    AccumuloConfiguration compactionConfig = getCompactionConfig(tablet.getTableConfiguration(), getOverrides(job.getKind(), tablet, cInfo.localHelper, job.getFiles()));
    FileCompactor compactor = new FileCompactor(tablet.getContext(), tablet.getExtent(), compactFiles, tmpFileName, cInfo.propagateDeletes, cenv, cInfo.iters, compactionConfig);
    return compactor.call();
}