Search in sources :

Example 1 with FileCompactor

use of org.apache.accumulo.server.compaction.FileCompactor in project accumulo by apache.

the class Compactor method createCompactionJob.

/**
 * Create compaction runnable
 *
 * @param job
 *          compaction job
 * @param totalInputEntries
 *          object to capture total entries
 * @param totalInputBytes
 *          object to capture input file size
 * @param started
 *          started latch
 * @param stopped
 *          stopped latch
 * @param err
 *          reference to error
 * @return Runnable compaction job
 */
protected Runnable createCompactionJob(final TExternalCompactionJob job, final LongAdder totalInputEntries, final LongAdder totalInputBytes, final CountDownLatch started, final CountDownLatch stopped, final AtomicReference<Throwable> err) {
    return new Runnable() {

        @Override
        public void run() {
            // Its only expected that a single compaction runs at a time. Multiple compactions running
            // at a time could cause odd behavior like out of order and unexpected thrift calls to the
            // coordinator. This is a sanity check to ensure the expectation is met. Should this check
            // ever fail, it means there is a bug elsewhere.
            Preconditions.checkState(compactionRunning.compareAndSet(false, true));
            try {
                LOG.info("Starting up compaction runnable for job: {}", job);
                TCompactionStatusUpdate update = new TCompactionStatusUpdate(TCompactionState.STARTED, "Compaction started", -1, -1, -1);
                updateCompactionState(job, update);
                final AccumuloConfiguration tConfig;
                var extent = KeyExtent.fromThrift(job.getExtent());
                if (!job.getOverrides().isEmpty()) {
                    tConfig = new ConfigurationCopy(getContext().getTableConfiguration(extent.tableId()));
                    job.getOverrides().forEach((k, v) -> ((ConfigurationCopy) tConfig).set(k, v));
                    LOG.debug("Overriding table properties with {}", job.getOverrides());
                } else {
                    tConfig = getContext().getTableConfiguration(extent.tableId());
                }
                final TabletFile outputFile = new TabletFile(new Path(job.getOutputFile()));
                final Map<StoredTabletFile, DataFileValue> files = new TreeMap<>();
                job.getFiles().forEach(f -> {
                    files.put(new StoredTabletFile(f.getMetadataFileEntry()), new DataFileValue(f.getSize(), f.getEntries(), f.getTimestamp()));
                    totalInputEntries.add(f.getEntries());
                    totalInputBytes.add(f.getSize());
                });
                final List<IteratorSetting> iters = new ArrayList<>();
                job.getIteratorSettings().getIterators().forEach(tis -> iters.add(SystemIteratorUtil.toIteratorSetting(tis)));
                ExtCEnv cenv = new ExtCEnv(JOB_HOLDER, queueName);
                FileCompactor compactor = new FileCompactor(getContext(), extent, files, outputFile, job.isPropagateDeletes(), cenv, iters, tConfig);
                LOG.trace("Starting compactor");
                started.countDown();
                org.apache.accumulo.server.compaction.CompactionStats stat = compactor.call();
                TCompactionStats cs = new TCompactionStats();
                cs.setEntriesRead(stat.getEntriesRead());
                cs.setEntriesWritten(stat.getEntriesWritten());
                cs.setFileSize(stat.getFileSize());
                JOB_HOLDER.setStats(cs);
                LOG.info("Compaction completed successfully {} ", job.getExternalCompactionId());
                // Update state when completed
                TCompactionStatusUpdate update2 = new TCompactionStatusUpdate(TCompactionState.SUCCEEDED, "Compaction completed successfully", -1, -1, -1);
                updateCompactionState(job, update2);
            } catch (Exception e) {
                LOG.error("Compaction failed", e);
                err.set(e);
            } finally {
                stopped.countDown();
                Preconditions.checkState(compactionRunning.compareAndSet(true, false));
            }
        }
    };
}
Also used : Path(org.apache.hadoop.fs.Path) ConfigurationCopy(org.apache.accumulo.core.conf.ConfigurationCopy) DataFileValue(org.apache.accumulo.core.metadata.schema.DataFileValue) TCompactionStatusUpdate(org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate) ArrayList(java.util.ArrayList) TreeMap(java.util.TreeMap) TTransportException(org.apache.thrift.transport.TTransportException) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) RetriesExceededException(org.apache.accumulo.server.compaction.RetryableThriftCall.RetriesExceededException) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) KeeperException(org.apache.zookeeper.KeeperException) TException(org.apache.thrift.TException) IOException(java.io.IOException) UnknownHostException(java.net.UnknownHostException) ThriftTableOperationException(org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException) UnknownCompactionIdException(org.apache.accumulo.core.compaction.thrift.UnknownCompactionIdException) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) TCompactionStats(org.apache.accumulo.core.tabletserver.thrift.TCompactionStats) FileCompactor(org.apache.accumulo.server.compaction.FileCompactor) StoredTabletFile(org.apache.accumulo.core.metadata.StoredTabletFile) TabletFile(org.apache.accumulo.core.metadata.TabletFile) StoredTabletFile(org.apache.accumulo.core.metadata.StoredTabletFile) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration)

Example 2 with FileCompactor

use of org.apache.accumulo.server.compaction.FileCompactor in project accumulo by apache.

the class CompactableUtils method compact.

/**
 * Create the FileCompactor and finally call compact. Returns the Major CompactionStats.
 */
static CompactionStats compact(Tablet tablet, CompactionJob job, CompactableImpl.CompactionInfo cInfo, CompactionEnv cenv, Map<StoredTabletFile, DataFileValue> compactFiles, TabletFile tmpFileName) throws IOException, CompactionCanceledException {
    AccumuloConfiguration compactionConfig = getCompactionConfig(tablet.getTableConfiguration(), getOverrides(job.getKind(), tablet, cInfo.localHelper, job.getFiles()));
    FileCompactor compactor = new FileCompactor(tablet.getContext(), tablet.getExtent(), compactFiles, tmpFileName, cInfo.propagateDeletes, cenv, cInfo.iters, compactionConfig);
    return compactor.call();
}
Also used : FileCompactor(org.apache.accumulo.server.compaction.FileCompactor) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration)

Aggregations

AccumuloConfiguration (org.apache.accumulo.core.conf.AccumuloConfiguration)2 FileCompactor (org.apache.accumulo.server.compaction.FileCompactor)2 IOException (java.io.IOException)1 UnknownHostException (java.net.UnknownHostException)1 ArrayList (java.util.ArrayList)1 TreeMap (java.util.TreeMap)1 AccumuloSecurityException (org.apache.accumulo.core.client.AccumuloSecurityException)1 IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)1 TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)1 ThriftSecurityException (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)1 ThriftTableOperationException (org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException)1 TCompactionStatusUpdate (org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate)1 UnknownCompactionIdException (org.apache.accumulo.core.compaction.thrift.UnknownCompactionIdException)1 ConfigurationCopy (org.apache.accumulo.core.conf.ConfigurationCopy)1 StoredTabletFile (org.apache.accumulo.core.metadata.StoredTabletFile)1 TabletFile (org.apache.accumulo.core.metadata.TabletFile)1 DataFileValue (org.apache.accumulo.core.metadata.schema.DataFileValue)1 TCompactionStats (org.apache.accumulo.core.tabletserver.thrift.TCompactionStats)1 RetriesExceededException (org.apache.accumulo.server.compaction.RetryableThriftCall.RetriesExceededException)1 Path (org.apache.hadoop.fs.Path)1