use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.
the class FileCompactor method compactLocalityGroup.
private void compactLocalityGroup(String lgName, Set<ByteSequence> columnFamilies, boolean inclusive, FileSKVWriter mfw, CompactionStats majCStats) throws IOException, CompactionCanceledException {
ArrayList<FileSKVIterator> readers = new ArrayList<>(filesToCompact.size());
Span compactSpan = TraceUtil.startSpan(this.getClass(), "compact");
try (Scope span = compactSpan.makeCurrent()) {
long entriesCompacted = 0;
List<SortedKeyValueIterator<Key, Value>> iters = openMapDataFiles(readers);
if (env.getIteratorScope() == IteratorScope.minc) {
iters.add(env.getMinCIterator());
}
CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()), entriesRead);
SortedKeyValueIterator<Key, Value> delIter = DeletingIterator.wrap(citr, propagateDeletes, DeletingIterator.getBehavior(acuTableConf));
ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
// if(env.getIteratorScope() )
SystemIteratorEnvironment iterEnv = env.createIteratorEnv(context, acuTableConf, getExtent().tableId());
SortedKeyValueIterator<Key, Value> itr = iterEnv.getTopLevelIterator(IterConfigUtil.convertItersAndLoad(env.getIteratorScope(), cfsi, acuTableConf, iterators, iterEnv));
itr.seek(extent.toDataRange(), columnFamilies, inclusive);
if (inclusive) {
mfw.startNewLocalityGroup(lgName, columnFamilies);
} else {
mfw.startDefaultLocalityGroup();
}
Span writeSpan = TraceUtil.startSpan(this.getClass(), "write");
try (Scope write = writeSpan.makeCurrent()) {
while (itr.hasTop() && env.isCompactionEnabled()) {
mfw.append(itr.getTopKey(), itr.getTopValue());
itr.next();
entriesCompacted++;
if (entriesCompacted % 1024 == 0) {
// Periodically update stats, do not want to do this too often since its volatile
entriesWritten.addAndGet(1024);
}
}
if (itr.hasTop() && !env.isCompactionEnabled()) {
// cancel major compaction operation
try {
try {
mfw.close();
} catch (IOException e) {
log.error("{}", e.getMessage(), e);
}
fs.deleteRecursively(outputFile.getPath());
} catch (Exception e) {
log.warn("Failed to delete Canceled compaction output file {}", outputFile, e);
}
throw new CompactionCanceledException();
}
} finally {
CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
majCStats.add(lgMajcStats);
writeSpan.end();
}
} catch (Exception e) {
TraceUtil.setException(compactSpan, e, true);
throw e;
} finally {
// close sequence files opened
for (FileSKVIterator reader : readers) {
try {
reader.close();
} catch (Exception e) {
log.warn("Failed to close map file", e);
}
}
compactSpan.end();
}
}
Aggregations