Search in sources :

Example 11 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class FileCompactor method compactLocalityGroup.

private void compactLocalityGroup(String lgName, Set<ByteSequence> columnFamilies, boolean inclusive, FileSKVWriter mfw, CompactionStats majCStats) throws IOException, CompactionCanceledException {
    ArrayList<FileSKVIterator> readers = new ArrayList<>(filesToCompact.size());
    Span compactSpan = TraceUtil.startSpan(this.getClass(), "compact");
    try (Scope span = compactSpan.makeCurrent()) {
        long entriesCompacted = 0;
        List<SortedKeyValueIterator<Key, Value>> iters = openMapDataFiles(readers);
        if (env.getIteratorScope() == IteratorScope.minc) {
            iters.add(env.getMinCIterator());
        }
        CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()), entriesRead);
        SortedKeyValueIterator<Key, Value> delIter = DeletingIterator.wrap(citr, propagateDeletes, DeletingIterator.getBehavior(acuTableConf));
        ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
        // if(env.getIteratorScope() )
        SystemIteratorEnvironment iterEnv = env.createIteratorEnv(context, acuTableConf, getExtent().tableId());
        SortedKeyValueIterator<Key, Value> itr = iterEnv.getTopLevelIterator(IterConfigUtil.convertItersAndLoad(env.getIteratorScope(), cfsi, acuTableConf, iterators, iterEnv));
        itr.seek(extent.toDataRange(), columnFamilies, inclusive);
        if (inclusive) {
            mfw.startNewLocalityGroup(lgName, columnFamilies);
        } else {
            mfw.startDefaultLocalityGroup();
        }
        Span writeSpan = TraceUtil.startSpan(this.getClass(), "write");
        try (Scope write = writeSpan.makeCurrent()) {
            while (itr.hasTop() && env.isCompactionEnabled()) {
                mfw.append(itr.getTopKey(), itr.getTopValue());
                itr.next();
                entriesCompacted++;
                if (entriesCompacted % 1024 == 0) {
                    // Periodically update stats, do not want to do this too often since its volatile
                    entriesWritten.addAndGet(1024);
                }
            }
            if (itr.hasTop() && !env.isCompactionEnabled()) {
                // cancel major compaction operation
                try {
                    try {
                        mfw.close();
                    } catch (IOException e) {
                        log.error("{}", e.getMessage(), e);
                    }
                    fs.deleteRecursively(outputFile.getPath());
                } catch (Exception e) {
                    log.warn("Failed to delete Canceled compaction output file {}", outputFile, e);
                }
                throw new CompactionCanceledException();
            }
        } finally {
            CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
            majCStats.add(lgMajcStats);
            writeSpan.end();
        }
    } catch (Exception e) {
        TraceUtil.setException(compactSpan, e, true);
        throw e;
    } finally {
        // close sequence files opened
        for (FileSKVIterator reader : readers) {
            try {
                reader.close();
            } catch (Exception e) {
                log.warn("Failed to close map file", e);
            }
        }
        compactSpan.end();
    }
}
Also used : FileSKVIterator(org.apache.accumulo.core.file.FileSKVIterator) ColumnFamilySkippingIterator(org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator) MultiIterator(org.apache.accumulo.core.iteratorsImpl.system.MultiIterator) ArrayList(java.util.ArrayList) SortedKeyValueIterator(org.apache.accumulo.core.iterators.SortedKeyValueIterator) IOException(java.io.IOException) Span(io.opentelemetry.api.trace.Span) IOException(java.io.IOException) IteratorScope(org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope) Scope(io.opentelemetry.context.Scope) Value(org.apache.accumulo.core.data.Value) DataFileValue(org.apache.accumulo.core.metadata.schema.DataFileValue) Key(org.apache.accumulo.core.data.Key) SystemIteratorEnvironment(org.apache.accumulo.server.iterators.SystemIteratorEnvironment)

Aggregations

ColumnFamilySkippingIterator (org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator)11 SortedMapIterator (org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator)8 Key (org.apache.accumulo.core.data.Key)7 Value (org.apache.accumulo.core.data.Value)7 ArrayByteSequence (org.apache.accumulo.core.data.ArrayByteSequence)6 ByteSequence (org.apache.accumulo.core.data.ByteSequence)6 Range (org.apache.accumulo.core.data.Range)6 Test (org.junit.jupiter.api.Test)5 HashSet (java.util.HashSet)4 TreeMap (java.util.TreeMap)4 ArrayList (java.util.ArrayList)3 IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)2 SortedKeyValueIterator (org.apache.accumulo.core.iterators.SortedKeyValueIterator)2 MultiIterator (org.apache.accumulo.core.iteratorsImpl.system.MultiIterator)2 Span (io.opentelemetry.api.trace.Span)1 Scope (io.opentelemetry.context.Scope)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 DataInputStream (java.io.DataInputStream)1 IOException (java.io.IOException)1