Examples with ServiceEnvironment org.apache.accumulo.core.spi.common.ServiceEnvironment used on opensource projects

Example 6 with ServiceEnvironment

use of org.apache.accumulo.core.spi.common.ServiceEnvironment in project accumulo by apache.

the class CompactableUtils method selectFiles.

static Set<StoredTabletFile> selectFiles(Tablet tablet, SortedMap<StoredTabletFile, DataFileValue> datafiles, PluginConfig selectorConfig) {
    CompactionSelector selector = newInstance(tablet.getTableConfiguration(), selectorConfig.getClassName(), CompactionSelector.class);
    final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());
    selector.init(new CompactionSelector.InitParameters() {

        @Override
        public Map<String, String> getOptions() {
            return selectorConfig.getOptions();
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }
    });
    Selection selection = selector.select(new CompactionSelector.SelectionParameters() {

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public Collection<CompactableFile> getAvailableFiles() {
            return Collections2.transform(datafiles.entrySet(), e -> new CompactableFileImpl(e.getKey(), e.getValue()));
        }

        @Override
        public Collection<Summary> getSummaries(Collection<CompactableFile> files, Predicate<SummarizerConfiguration> summarySelector) {
            var context = tablet.getContext();
            var tsrm = tablet.getTabletResources().getTabletServerResourceManager();
            SummaryCollection sc = new SummaryCollection();
            SummarizerFactory factory = new SummarizerFactory(tablet.getTableConfiguration());
            for (CompactableFile cf : files) {
                var file = CompactableFileImpl.toStoredTabletFile(cf);
                FileSystem fs = context.getVolumeManager().getFileSystemByPath(file.getPath());
                Configuration conf = context.getHadoopConf();
                SummaryCollection fsc = SummaryReader.load(fs, conf, factory, file.getPath(), summarySelector, tsrm.getSummaryCache(), tsrm.getIndexCache(), tsrm.getFileLenCache(), context.getCryptoService()).getSummaries(Collections.singletonList(new Gatherer.RowRange(tablet.getExtent())));
                sc.merge(fsc, factory);
            }
            return sc.getSummaries();
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }

        @Override
        public Optional<SortedKeyValueIterator<Key, Value>> getSample(CompactableFile file, SamplerConfiguration sc) {
            try {
                FileOperations fileFactory = FileOperations.getInstance();
                Path path = new Path(file.getUri());
                FileSystem ns = tablet.getTabletServer().getVolumeManager().getFileSystemByPath(path);
                var fiter = fileFactory.newReaderBuilder().forFile(path.toString(), ns, ns.getConf(), tablet.getContext().getCryptoService()).withTableConfiguration(tablet.getTableConfiguration()).seekToBeginning().build();
                return Optional.ofNullable(fiter.getSample(new SamplerConfigurationImpl(sc)));
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }
    });
    return selection.getFilesToCompact().stream().map(CompactableFileImpl::toStoredTabletFile).collect(Collectors.toSet());
}

Example 7 with ServiceEnvironment

use of org.apache.accumulo.core.spi.common.ServiceEnvironment in project accumulo by apache.

the class IteratorEnvIT method testEnv.

/**
 * Test the environment methods return what is expected.
 */
private static void testEnv(IteratorScope scope, Map<String, String> opts, IteratorEnvironment env) {
    TableId expectedTableId = TableId.of(opts.get("expected.table.id"));
    // verify getServiceEnv() and getPluginEnv() are the same objects,
    // so further checks only need to use getPluginEnv()
    @SuppressWarnings("deprecation") ServiceEnvironment serviceEnv = env.getServiceEnv();
    PluginEnvironment pluginEnv = env.getPluginEnv();
    if (serviceEnv != pluginEnv)
        throw new RuntimeException("Test failed - assertSame(getServiceEnv(),getPluginEnv())");
    // verify property exists on the table config (deprecated and new),
    // with and without custom prefix, but not in the system config
    @SuppressWarnings("deprecation") String accTableConf = env.getConfig().get("table.custom.iterator.env.test");
    if (!"value1".equals(accTableConf))
        throw new RuntimeException("Test failed - Expected table property not found in getConfig().");
    var tableConf = pluginEnv.getConfiguration(env.getTableId());
    if (!"value1".equals(tableConf.get("table.custom.iterator.env.test")))
        throw new RuntimeException("Test failed - Expected table property not found in table conf.");
    if (!"value1".equals(tableConf.getTableCustom("iterator.env.test")))
        throw new RuntimeException("Test failed - Expected table property not found in table conf.");
    var systemConf = pluginEnv.getConfiguration();
    if (systemConf.get("table.custom.iterator.env.test") != null)
        throw new RuntimeException("Test failed - Unexpected table property found in system conf.");
    // check other environment settings
    if (!scope.equals(env.getIteratorScope()))
        throw new RuntimeException("Test failed - Error getting iterator scope");
    if (env.isSamplingEnabled())
        throw new RuntimeException("Test failed - isSamplingEnabled returned true, expected false");
    if (!expectedTableId.equals(env.getTableId()))
        throw new RuntimeException("Test failed - Error getting Table ID");
}

Example 8 with ServiceEnvironment

use of org.apache.accumulo.core.spi.common.ServiceEnvironment in project accumulo by apache.

the class CompactableImpl method getConfiguredService.

@Override
public CompactionServiceId getConfiguredService(CompactionKind kind) {
    Map<String, String> debugHints = null;
    try {
        var dispatcher = tablet.getTableConfiguration().getCompactionDispatcher();
        Map<String, String> tmpHints = Map.of();
        if (kind == CompactionKind.USER) {
            synchronized (this) {
                if (fileMgr.getSelectionStatus() != FileSelectionStatus.NOT_ACTIVE && fileMgr.getSelectionStatus() != FileSelectionStatus.CANCELED && fileMgr.getSelectionKind() == CompactionKind.USER) {
                    tmpHints = compactionConfig.getExecutionHints();
                }
            }
        }
        var hints = tmpHints;
        debugHints = hints;
        var dispatch = dispatcher.dispatch(new DispatchParameters() {

            private final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());

            @Override
            public ServiceEnvironment getServiceEnv() {
                return senv;
            }

            @Override
            public Map<String, String> getExecutionHints() {
                return hints;
            }

            @Override
            public CompactionKind getCompactionKind() {
                return kind;
            }

            @Override
            public CompactionServices getCompactionServices() {
                return manager.getServices();
            }
        });
        return dispatch.getService();
    } catch (RuntimeException e) {
        log.error("Failed to dispatch compaction {} kind:{} hints:{}, falling back to {} service.", getExtent(), kind, debugHints, CompactionServicesConfig.DEFAULT_SERVICE, e);
        return CompactionServicesConfig.DEFAULT_SERVICE;
    }
}

Example 9 with ServiceEnvironment

use of org.apache.accumulo.core.spi.common.ServiceEnvironment in project accumulo by apache.

the class CompactableUtils method computeOverrides.

static Map<String, String> computeOverrides(Tablet tablet, Set<CompactableFile> files, PluginConfig cfg) {
    CompactionConfigurer configurer = CompactableUtils.newInstance(tablet.getTableConfiguration(), cfg.getClassName(), CompactionConfigurer.class);
    final ServiceEnvironment senv = new ServiceEnvironmentImpl(tablet.getContext());
    configurer.init(new CompactionConfigurer.InitParameters() {

        @Override
        public Map<String, String> getOptions() {
            return cfg.getOptions();
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }
    });
    var overrides = configurer.override(new CompactionConfigurer.InputParameters() {

        @Override
        public Collection<CompactableFile> getInputFiles() {
            return files;
        }

        @Override
        public PluginEnvironment getEnvironment() {
            return senv;
        }

        @Override
        public TableId getTableId() {
            return tablet.getExtent().tableId();
        }
    });
    if (overrides.getOverrides().isEmpty()) {
        return null;
    }
    return overrides.getOverrides();
}

Example 10 with ServiceEnvironment

use of org.apache.accumulo.core.spi.common.ServiceEnvironment in project accumulo by apache.

the class TabletServerResourceManager method createPriorityExecutor.

private ThreadPoolExecutor createPriorityExecutor(ScanExecutorConfig sec, Map<String, Queue<Runnable>> scanExecQueues) {
    BlockingQueue<Runnable> queue;
    if (sec.prioritizerClass.orElse("").isEmpty()) {
        queue = new LinkedBlockingQueue<>();
    } else {
        ScanPrioritizer factory = null;
        try {
            factory = ConfigurationTypeHelper.getClassInstance(null, sec.prioritizerClass.get(), ScanPrioritizer.class);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        if (factory == null) {
            queue = new LinkedBlockingQueue<>();
        } else {
            Comparator<ScanInfo> comparator = factory.createComparator(new ScanPrioritizer.CreateParameters() {

                private final ServiceEnvironment senv = new ServiceEnvironmentImpl(context);

                @Override
                public Map<String, String> getOptions() {
                    return sec.prioritizerOpts;
                }

                @Override
                public ServiceEnvironment getServiceEnv() {
                    return senv;
                }
            });
            // function to extract scan session from runnable
            Function<Runnable, ScanInfo> extractor = r -> ((ScanSession.ScanMeasurer) TraceUtil.unwrap(r)).getScanInfo();
            queue = new PriorityBlockingQueue<>(sec.maxThreads, Comparator.comparing(extractor, comparator));
        }
    }
    scanExecQueues.put(sec.name, queue);
    ThreadPoolExecutor es = ThreadPools.createThreadPool(sec.getCurrentMaxThreads(), sec.getCurrentMaxThreads(), 0L, TimeUnit.MILLISECONDS, "scan-" + sec.name, queue, sec.priority, true);
    modifyThreadPoolSizesAtRuntime(sec::getCurrentMaxThreads, "scan-" + sec.name, es);
    return es;
}