use of org.apache.accumulo.tserver.compaction.MajorCompactionRequest in project accumulo by apache.
the class TooManyDeletesCompactionStrategy method gatherInformation.
@Override
public void gatherInformation(MajorCompactionRequest request) throws IOException {
super.gatherInformation(request);
Predicate<SummarizerConfiguration> summarizerPredicate = conf -> conf.getClassName().equals(DeletesSummarizer.class.getName()) && conf.getOptions().isEmpty();
long total = 0;
long deletes = 0;
for (Entry<FileRef, DataFileValue> entry : request.getFiles().entrySet()) {
Collection<Summary> summaries = request.getSummaries(Collections.singleton(entry.getKey()), summarizerPredicate);
if (summaries.size() == 1) {
Summary summary = summaries.iterator().next();
total += summary.getStatistics().get(TOTAL_STAT);
deletes += summary.getStatistics().get(DELETES_STAT);
} else {
long numEntries = entry.getValue().getNumEntries();
if (numEntries == 0 && !proceed_bns) {
shouldCompact = false;
return;
} else {
// no summary data so use Accumulo's estimate of total entries in file
total += entry.getValue().getNumEntries();
}
}
}
long nonDeletes = total - deletes;
if (nonDeletes >= 0) {
// check nonDeletes >= 0 because if this is not true then its clear evidence that the estimates are off
double ratio = deletes / (double) nonDeletes;
shouldCompact = ratio >= threshold;
} else {
shouldCompact = false;
}
}
use of org.apache.accumulo.tserver.compaction.MajorCompactionRequest in project accumulo by apache.
the class Tablet method _majorCompact.
private CompactionStats _majorCompact(MajorCompactionReason reason) throws IOException, CompactionCanceledException {
long t1, t2, t3;
Pair<Long, UserCompactionConfig> compactionId = null;
CompactionStrategy strategy = null;
Map<FileRef, Pair<Key, Key>> firstAndLastKeys = null;
if (reason == MajorCompactionReason.USER) {
try {
compactionId = getCompactionID();
strategy = createCompactionStrategy(compactionId.getSecond().getCompactionStrategy());
} catch (NoNodeException e) {
throw new RuntimeException(e);
}
} else if (reason == MajorCompactionReason.NORMAL || reason == MajorCompactionReason.IDLE) {
strategy = Property.createTableInstanceFromPropertyName(tableConfiguration, Property.TABLE_COMPACTION_STRATEGY, CompactionStrategy.class, new DefaultCompactionStrategy());
strategy.init(Property.getCompactionStrategyOptions(tableConfiguration));
} else if (reason == MajorCompactionReason.CHOP) {
firstAndLastKeys = getFirstAndLastKeys(getDatafileManager().getDatafileSizes());
} else {
throw new IllegalArgumentException("Unknown compaction reason " + reason);
}
if (strategy != null) {
BlockCache sc = tabletResources.getTabletServerResourceManager().getSummaryCache();
BlockCache ic = tabletResources.getTabletServerResourceManager().getIndexCache();
MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, getTabletServer().getFileSystem(), tableConfiguration, sc, ic);
request.setFiles(getDatafileManager().getDatafileSizes());
strategy.gatherInformation(request);
}
Map<FileRef, DataFileValue> filesToCompact = null;
int maxFilesToCompact = tableConfiguration.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
CompactionStats majCStats = new CompactionStats();
CompactionPlan plan = null;
boolean propogateDeletes = false;
boolean updateCompactionID = false;
synchronized (this) {
// plan all that work that needs to be done in the sync block... then do the actual work
// outside the sync block
t1 = System.currentTimeMillis();
majorCompactionState = CompactionState.WAITING_TO_START;
getTabletMemory().waitForMinC();
t2 = System.currentTimeMillis();
majorCompactionState = CompactionState.IN_PROGRESS;
notifyAll();
VolumeManager fs = getTabletServer().getFileSystem();
if (extent.isRootTablet()) {
// very important that we call this before doing major compaction,
// otherwise deleted compacted files could possible be brought back
// at some point if the file they were compacted to was legitimately
// removed by a major compaction
RootFiles.cleanupReplacement(fs, fs.listStatus(this.location), false);
}
SortedMap<FileRef, DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
List<FileRef> inputFiles = new ArrayList<>();
if (reason == MajorCompactionReason.CHOP) {
// enforce rules: files with keys outside our range need to be compacted
inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
} else {
MajorCompactionRequest request = new MajorCompactionRequest(extent, reason, tableConfiguration);
request.setFiles(allFiles);
plan = strategy.getCompactionPlan(request);
if (plan != null) {
plan.validate(allFiles.keySet());
inputFiles.addAll(plan.inputFiles);
}
}
if (inputFiles.isEmpty()) {
if (reason == MajorCompactionReason.USER) {
if (compactionId.getSecond().getIterators().isEmpty()) {
log.debug("No-op major compaction by USER on 0 input files because no iterators present.");
lastCompactID = compactionId.getFirst();
updateCompactionID = true;
} else {
log.debug("Major compaction by USER on 0 input files with iterators.");
filesToCompact = new HashMap<>();
}
} else {
return majCStats;
}
} else {
// If no original files will exist at the end of the compaction, we do not have to propogate deletes
Set<FileRef> droppedFiles = new HashSet<>();
droppedFiles.addAll(inputFiles);
if (plan != null)
droppedFiles.addAll(plan.deleteFiles);
propogateDeletes = !(droppedFiles.equals(allFiles.keySet()));
log.debug("Major compaction plan: {} propogate deletes : {}", plan, propogateDeletes);
filesToCompact = new HashMap<>(allFiles);
filesToCompact.keySet().retainAll(inputFiles);
getDatafileManager().reserveMajorCompactingFiles(filesToCompact.keySet());
}
t3 = System.currentTimeMillis();
}
try {
log.debug(String.format("MajC initiate lock %.2f secs, wait %.2f secs", (t3 - t2) / 1000.0, (t2 - t1) / 1000.0));
if (updateCompactionID) {
MetadataTableUtil.updateTabletCompactID(extent, compactionId.getFirst(), tabletServer, getTabletServer().getLock());
return majCStats;
}
if (!propogateDeletes && compactionId == null) {
// compacting everything, so update the compaction id in metadata
try {
compactionId = getCompactionID();
if (compactionId.getSecond().getCompactionStrategy() != null) {
compactionId = null;
// TODO maybe return unless chop?
}
} catch (NoNodeException e) {
throw new RuntimeException(e);
}
}
List<IteratorSetting> compactionIterators = new ArrayList<>();
if (compactionId != null) {
if (reason == MajorCompactionReason.USER) {
if (getCompactionCancelID() >= compactionId.getFirst()) {
// compaction was canceled
return majCStats;
}
compactionIterators = compactionId.getSecond().getIterators();
synchronized (this) {
if (lastCompactID >= compactionId.getFirst())
// already compacted
return majCStats;
}
}
}
// ACCUMULO-3645 run loop at least once, even if filesToCompact.isEmpty()
do {
int numToCompact = maxFilesToCompact;
if (filesToCompact.size() > maxFilesToCompact && filesToCompact.size() < 2 * maxFilesToCompact) {
// on the second to last compaction pass, compact the minimum amount of files possible
numToCompact = filesToCompact.size() - maxFilesToCompact + 1;
}
Set<FileRef> smallestFiles = removeSmallest(filesToCompact, numToCompact);
FileRef fileName = getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
FileRef compactTmpName = new FileRef(fileName.path().toString() + "_tmp");
AccumuloConfiguration tableConf = createTableConfiguration(tableConfiguration, plan);
Span span = Trace.start("compactFiles");
try {
CompactionEnv cenv = new CompactionEnv() {
@Override
public boolean isCompactionEnabled() {
return Tablet.this.isCompactionEnabled();
}
@Override
public IteratorScope getIteratorScope() {
return IteratorScope.majc;
}
@Override
public RateLimiter getReadLimiter() {
return getTabletServer().getMajorCompactionReadLimiter();
}
@Override
public RateLimiter getWriteLimiter() {
return getTabletServer().getMajorCompactionWriteLimiter();
}
};
HashMap<FileRef, DataFileValue> copy = new HashMap<>(getDatafileManager().getDatafileSizes());
if (!copy.keySet().containsAll(smallestFiles))
throw new IllegalStateException("Cannot find data file values for " + smallestFiles);
copy.keySet().retainAll(smallestFiles);
log.debug("Starting MajC {} ({}) {} --> {} {}", extent, reason, copy.keySet(), compactTmpName, compactionIterators);
// always propagate deletes, unless last batch
boolean lastBatch = filesToCompact.isEmpty();
Compactor compactor = new Compactor(tabletServer, this, copy, null, compactTmpName, lastBatch ? propogateDeletes : true, cenv, compactionIterators, reason.ordinal(), tableConf);
CompactionStats mcs = compactor.call();
span.data("files", "" + smallestFiles.size());
span.data("read", "" + mcs.getEntriesRead());
span.data("written", "" + mcs.getEntriesWritten());
majCStats.add(mcs);
if (lastBatch && plan != null && plan.deleteFiles != null) {
smallestFiles.addAll(plan.deleteFiles);
}
getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName, filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
// to add the deleted file
if (filesToCompact.size() > 0 && mcs.getEntriesWritten() > 0) {
filesToCompact.put(fileName, new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
}
} finally {
span.stop();
}
} while (filesToCompact.size() > 0);
return majCStats;
} finally {
synchronized (Tablet.this) {
getDatafileManager().clearMajorCompactingFile();
}
}
}
use of org.apache.accumulo.tserver.compaction.MajorCompactionRequest in project accumulo by apache.
the class Tablet method compactAll.
public void compactAll(long compactionId, UserCompactionConfig compactionConfig) {
boolean updateMetadata = false;
synchronized (this) {
if (lastCompactID >= compactionId)
return;
if (isMinorCompactionRunning()) {
// want to wait for running minc to finish before starting majc, see ACCUMULO-3041
if (compactionWaitInfo.compactionID == compactionId) {
if (lastFlushID == compactionWaitInfo.flushID)
return;
} else {
compactionWaitInfo.compactionID = compactionId;
compactionWaitInfo.flushID = lastFlushID;
return;
}
}
if (isClosing() || isClosed() || majorCompactionQueued.contains(MajorCompactionReason.USER) || isMajorCompactionRunning())
return;
CompactionStrategyConfig strategyConfig = compactionConfig.getCompactionStrategy();
CompactionStrategy strategy = createCompactionStrategy(strategyConfig);
MajorCompactionRequest request = new MajorCompactionRequest(extent, MajorCompactionReason.USER, tableConfiguration);
request.setFiles(getDatafileManager().getDatafileSizes());
try {
if (strategy.shouldCompact(request)) {
initiateMajorCompaction(MajorCompactionReason.USER);
} else {
majorCompactionState = CompactionState.IN_PROGRESS;
updateMetadata = true;
lastCompactID = compactionId;
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (updateMetadata) {
try {
// if multiple threads were allowed to update this outside of a sync block, then it would be
// a race condition
MetadataTableUtil.updateTabletCompactID(extent, compactionId, getTabletServer(), getTabletServer().getLock());
} finally {
synchronized (this) {
majorCompactionState = null;
this.notifyAll();
}
}
}
}
use of org.apache.accumulo.tserver.compaction.MajorCompactionRequest in project accumulo by apache.
the class ConfigurableCompactionStrategyTest method testOutputOptions.
// file selection options are adequately tested by ShellServerIT
@Test
public void testOutputOptions() throws Exception {
MajorCompactionRequest mcr = new MajorCompactionRequest(new KeyExtent(Table.ID.of("1"), null, null), MajorCompactionReason.USER, null);
Map<FileRef, DataFileValue> files = new HashMap<>();
files.put(new FileRef("hdfs://nn1/accumulo/tables/1/t-009/F00001.rf"), new DataFileValue(50000, 400));
mcr.setFiles(files);
// test setting no output options
ConfigurableCompactionStrategy ccs = new ConfigurableCompactionStrategy();
Map<String, String> opts = new HashMap<>();
ccs.init(opts);
CompactionPlan plan = ccs.getCompactionPlan(mcr);
Assert.assertEquals(0, plan.writeParameters.getBlockSize());
Assert.assertEquals(0, plan.writeParameters.getHdfsBlockSize());
Assert.assertEquals(0, plan.writeParameters.getIndexBlockSize());
Assert.assertEquals(0, plan.writeParameters.getReplication());
Assert.assertEquals(null, plan.writeParameters.getCompressType());
// test setting all output options
ccs = new ConfigurableCompactionStrategy();
CompactionSettings.OUTPUT_BLOCK_SIZE_OPT.put(opts, "64K");
CompactionSettings.OUTPUT_COMPRESSION_OPT.put(opts, "snappy");
CompactionSettings.OUTPUT_HDFS_BLOCK_SIZE_OPT.put(opts, "256M");
CompactionSettings.OUTPUT_INDEX_BLOCK_SIZE_OPT.put(opts, "32K");
CompactionSettings.OUTPUT_REPLICATION_OPT.put(opts, "5");
ccs.init(opts);
plan = ccs.getCompactionPlan(mcr);
Assert.assertEquals(ConfigurationTypeHelper.getFixedMemoryAsBytes("64K"), plan.writeParameters.getBlockSize());
Assert.assertEquals(ConfigurationTypeHelper.getFixedMemoryAsBytes("256M"), plan.writeParameters.getHdfsBlockSize());
Assert.assertEquals(ConfigurationTypeHelper.getFixedMemoryAsBytes("32K"), plan.writeParameters.getIndexBlockSize());
Assert.assertEquals(5, plan.writeParameters.getReplication());
Assert.assertEquals("snappy", plan.writeParameters.getCompressType());
}
Aggregations