Search in sources :

Example 1 with CompactableFileImpl

use of org.apache.accumulo.core.metadata.CompactableFileImpl in project accumulo by apache.

the class CompactableUtils method selectFiles.

static Set<StoredTabletFile> selectFiles(Tablet tablet, SortedMap<StoredTabletFile, DataFileValue> datafiles, PluginConfig selectorConfig) {
    CompactionSelector selector = newInstance(tablet.getTableConfiguration(), selectorConfig.getClassName(), CompactionSelector.class);
    final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());
    selector.init(new CompactionSelector.InitParameters() {

        @Override
        public Map<String, String> getOptions() {
            return selectorConfig.getOptions();
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }
    });
    Selection selection = selector.select(new CompactionSelector.SelectionParameters() {

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public Collection<CompactableFile> getAvailableFiles() {
            return Collections2.transform(datafiles.entrySet(), e -> new CompactableFileImpl(e.getKey(), e.getValue()));
        }

        @Override
        public Collection<Summary> getSummaries(Collection<CompactableFile> files, Predicate<SummarizerConfiguration> summarySelector) {
            var context = tablet.getContext();
            var tsrm = tablet.getTabletResources().getTabletServerResourceManager();
            SummaryCollection sc = new SummaryCollection();
            SummarizerFactory factory = new SummarizerFactory(tablet.getTableConfiguration());
            for (CompactableFile cf : files) {
                var file = CompactableFileImpl.toStoredTabletFile(cf);
                FileSystem fs = context.getVolumeManager().getFileSystemByPath(file.getPath());
                Configuration conf = context.getHadoopConf();
                SummaryCollection fsc = SummaryReader.load(fs, conf, factory, file.getPath(), summarySelector, tsrm.getSummaryCache(), tsrm.getIndexCache(), tsrm.getFileLenCache(), context.getCryptoService()).getSummaries(Collections.singletonList(new Gatherer.RowRange(tablet.getExtent())));
                sc.merge(fsc, factory);
            }
            return sc.getSummaries();
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }

        @Override
        public Optional<SortedKeyValueIterator<Key, Value>> getSample(CompactableFile file, SamplerConfiguration sc) {
            try {
                FileOperations fileFactory = FileOperations.getInstance();
                Path path = new Path(file.getUri());
                FileSystem ns = tablet.getTabletServer().getVolumeManager().getFileSystemByPath(path);
                var fiter = fileFactory.newReaderBuilder().forFile(path.toString(), ns, ns.getConf(), tablet.getContext().getCryptoService()).withTableConfiguration(tablet.getTableConfiguration()).seekToBeginning().build();
                return Optional.ofNullable(fiter.getSample(new SamplerConfigurationImpl(sc)));
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }
    });
    return selection.getFilesToCompact().stream().map(CompactableFileImpl::toStoredTabletFile).collect(Collectors.toSet());
}
Also used : PluginEnvironment(org.apache.accumulo.core.client.PluginEnvironment) TableId(org.apache.accumulo.core.data.TableId) TableId(org.apache.accumulo.core.data.TableId) FileCompactor(org.apache.accumulo.server.compaction.FileCompactor) FileSystem(org.apache.hadoop.fs.FileSystem) LoggerFactory(org.slf4j.LoggerFactory) Collections2(com.google.common.collect.Collections2) MetadataTableUtil(org.apache.accumulo.server.util.MetadataTableUtil) Selection(org.apache.accumulo.core.client.admin.compaction.CompactionSelector.Selection) PluginConfig(org.apache.accumulo.core.client.admin.PluginConfig) CompactableFile(org.apache.accumulo.core.client.admin.compaction.CompactableFile) BlockCache(org.apache.accumulo.core.spi.cache.BlockCache) ConfigurationTypeHelper(org.apache.accumulo.core.conf.ConfigurationTypeHelper) ClassLoaderUtil(org.apache.accumulo.core.classloader.ClassLoaderUtil) FileOperations(org.apache.accumulo.core.file.FileOperations) Map(java.util.Map) Configuration(org.apache.hadoop.conf.Configuration) Path(org.apache.hadoop.fs.Path) CompactionStrategyConfigUtil(org.apache.accumulo.core.clientImpl.CompactionStrategyConfigUtil) Value(org.apache.accumulo.core.data.Value) SamplerConfiguration(org.apache.accumulo.core.client.sample.SamplerConfiguration) Property(org.apache.accumulo.core.conf.Property) CompactionEnv(org.apache.accumulo.server.compaction.FileCompactor.CompactionEnv) VolumeManager(org.apache.accumulo.server.fs.VolumeManager) Predicate(java.util.function.Predicate) Collection(java.util.Collection) MajorCompactionReason(org.apache.accumulo.tserver.compaction.MajorCompactionReason) FileSKVIterator(org.apache.accumulo.core.file.FileSKVIterator) Set(java.util.Set) SummaryReader(org.apache.accumulo.core.summary.SummaryReader) Collectors(java.util.stream.Collectors) UncheckedIOException(java.io.UncheckedIOException) CompactionStrategyConfig(org.apache.accumulo.core.client.admin.CompactionStrategyConfig) TableConfiguration(org.apache.accumulo.server.conf.TableConfiguration) CompactionSelector(org.apache.accumulo.core.client.admin.compaction.CompactionSelector) Pair(org.apache.accumulo.core.util.Pair) ServiceEnvironment(org.apache.accumulo.core.spi.common.ServiceEnvironment) Optional(java.util.Optional) ConfigurationCopy(org.apache.accumulo.core.conf.ConfigurationCopy) CacheBuilder(com.google.common.cache.CacheBuilder) SortedMap(java.util.SortedMap) SummarizerFactory(org.apache.accumulo.core.summary.SummarizerFactory) CompactionPlan(org.apache.accumulo.tserver.compaction.CompactionPlan) DataFileValue(org.apache.accumulo.core.metadata.schema.DataFileValue) SamplerConfigurationImpl(org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl) CompactionHelper(org.apache.accumulo.tserver.tablet.CompactableImpl.CompactionHelper) SummarizerConfiguration(org.apache.accumulo.core.client.summary.SummarizerConfiguration) SummaryCollection(org.apache.accumulo.core.summary.SummaryCollection) HashMap(java.util.HashMap) CompactionConfig(org.apache.accumulo.core.client.admin.CompactionConfig) SortedKeyValueIterator(org.apache.accumulo.core.iterators.SortedKeyValueIterator) PluginEnvironment(org.apache.accumulo.core.client.PluginEnvironment) HashSet(java.util.HashSet) StoredTabletFile(org.apache.accumulo.core.metadata.StoredTabletFile) Gatherer(org.apache.accumulo.core.summary.Gatherer) CompactionKind(org.apache.accumulo.core.spi.compaction.CompactionKind) Key(org.apache.accumulo.core.data.Key) ServiceEnvironmentImpl(org.apache.accumulo.server.ServiceEnvironmentImpl) TabletFile(org.apache.accumulo.core.metadata.TabletFile) CompactionStats(org.apache.accumulo.server.compaction.CompactionStats) Summary(org.apache.accumulo.core.client.summary.Summary) CompactionCanceledException(org.apache.accumulo.server.compaction.FileCompactor.CompactionCanceledException) Logger(org.slf4j.Logger) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) MajorCompactionRequest(org.apache.accumulo.tserver.compaction.MajorCompactionRequest) IOException(java.io.IOException) CompactionConfigurer(org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration) WriteParameters(org.apache.accumulo.tserver.compaction.WriteParameters) ExecutionException(java.util.concurrent.ExecutionException) CompactionJob(org.apache.accumulo.core.spi.compaction.CompactionJob) Preconditions(com.google.common.base.Preconditions) CompactionStrategy(org.apache.accumulo.tserver.compaction.CompactionStrategy) Cache(com.google.common.cache.Cache) Collections(java.util.Collections) UserCompactionUtils(org.apache.accumulo.core.clientImpl.UserCompactionUtils) CompactableFileImpl(org.apache.accumulo.core.metadata.CompactableFileImpl) Gatherer(org.apache.accumulo.core.summary.Gatherer) Configuration(org.apache.hadoop.conf.Configuration) SamplerConfiguration(org.apache.accumulo.core.client.sample.SamplerConfiguration) TableConfiguration(org.apache.accumulo.server.conf.TableConfiguration) SummarizerConfiguration(org.apache.accumulo.core.client.summary.SummarizerConfiguration) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration) Selection(org.apache.accumulo.core.client.admin.compaction.CompactionSelector.Selection) SamplerConfigurationImpl(org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl) SamplerConfiguration(org.apache.accumulo.core.client.sample.SamplerConfiguration) FileOperations(org.apache.accumulo.core.file.FileOperations) SummarizerFactory(org.apache.accumulo.core.summary.SummarizerFactory) UncheckedIOException(java.io.UncheckedIOException) ServiceEnvironment(org.apache.accumulo.core.spi.common.ServiceEnvironment) FileSystem(org.apache.hadoop.fs.FileSystem) SummarizerConfiguration(org.apache.accumulo.core.client.summary.SummarizerConfiguration) Path(org.apache.hadoop.fs.Path) CompactableFileImpl(org.apache.accumulo.core.metadata.CompactableFileImpl) CompactableFile(org.apache.accumulo.core.client.admin.compaction.CompactableFile) Optional(java.util.Optional) ServiceEnvironmentImpl(org.apache.accumulo.server.ServiceEnvironmentImpl) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) CompactionSelector(org.apache.accumulo.core.client.admin.compaction.CompactionSelector) Value(org.apache.accumulo.core.data.Value) DataFileValue(org.apache.accumulo.core.metadata.schema.DataFileValue) Collection(java.util.Collection) SummaryCollection(org.apache.accumulo.core.summary.SummaryCollection) Map(java.util.Map) SortedMap(java.util.SortedMap) HashMap(java.util.HashMap) SummaryCollection(org.apache.accumulo.core.summary.SummaryCollection) Key(org.apache.accumulo.core.data.Key)

Aggregations

Preconditions (com.google.common.base.Preconditions)1 Cache (com.google.common.cache.Cache)1 CacheBuilder (com.google.common.cache.CacheBuilder)1 Collections2 (com.google.common.collect.Collections2)1 IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Set (java.util.Set)1 SortedMap (java.util.SortedMap)1 ExecutionException (java.util.concurrent.ExecutionException)1 Predicate (java.util.function.Predicate)1 Collectors (java.util.stream.Collectors)1 ClassLoaderUtil (org.apache.accumulo.core.classloader.ClassLoaderUtil)1 PluginEnvironment (org.apache.accumulo.core.client.PluginEnvironment)1 CompactionConfig (org.apache.accumulo.core.client.admin.CompactionConfig)1