Examples with ScanExecutorConfig org.apache.accumulo.core.conf.AccumuloConfiguration.ScanExecutorConfig used on opensource projects

Example 1 with ScanExecutorConfig

use of org.apache.accumulo.core.conf.AccumuloConfiguration.ScanExecutorConfig in project accumulo by apache.

the class AccumuloConfigurationTest method testScanExecutors.

@Test
public void testScanExecutors() {
    String defName = SimpleScanDispatcher.DEFAULT_SCAN_EXECUTOR_NAME;
    TestConfiguration tc = new TestConfiguration(DefaultConfiguration.getInstance());
    Collection<ScanExecutorConfig> executors = tc.getScanExecutors();
    assertEquals(2, executors.size());
    ScanExecutorConfig sec = executors.stream().filter(c -> c.name.equals(defName)).findFirst().get();
    assertEquals(Integer.parseInt(Property.TSERV_SCAN_EXECUTORS_DEFAULT_THREADS.getDefaultValue()), sec.maxThreads);
    assertFalse(sec.priority.isPresent());
    assertTrue(sec.prioritizerClass.get().isEmpty());
    assertTrue(sec.prioritizerOpts.isEmpty());
    // ensure deprecated props is read if nothing else is set
    tc.set("tserver.readahead.concurrent.max", "6");
    assertEquals(6, sec.getCurrentMaxThreads());
    assertEquals(Integer.parseInt(Property.TSERV_SCAN_EXECUTORS_DEFAULT_THREADS.getDefaultValue()), sec.maxThreads);
    ScanExecutorConfig sec2 = tc.getScanExecutors().stream().filter(c -> c.name.equals(defName)).findFirst().get();
    assertEquals(6, sec2.maxThreads);
    // ensure new prop overrides deprecated prop
    tc.set(Property.TSERV_SCAN_EXECUTORS_DEFAULT_THREADS.getKey(), "9");
    assertEquals(9, sec.getCurrentMaxThreads());
    assertEquals(Integer.parseInt(Property.TSERV_SCAN_EXECUTORS_DEFAULT_THREADS.getDefaultValue()), sec.maxThreads);
    ScanExecutorConfig sec3 = tc.getScanExecutors().stream().filter(c -> c.name.equals(defName)).findFirst().get();
    assertEquals(9, sec3.maxThreads);
    ScanExecutorConfig sec4 = executors.stream().filter(c -> c.name.equals("meta")).findFirst().get();
    assertEquals(Integer.parseInt(Property.TSERV_SCAN_EXECUTORS_META_THREADS.getDefaultValue()), sec4.maxThreads);
    assertFalse(sec4.priority.isPresent());
    assertFalse(sec4.prioritizerClass.isPresent());
    assertTrue(sec4.prioritizerOpts.isEmpty());
    tc.set("tserver.metadata.readahead.concurrent.max", "2");
    assertEquals(2, sec4.getCurrentMaxThreads());
    ScanExecutorConfig sec5 = tc.getScanExecutors().stream().filter(c -> c.name.equals("meta")).findFirst().get();
    assertEquals(2, sec5.maxThreads);
    tc.set(Property.TSERV_SCAN_EXECUTORS_META_THREADS.getKey(), "3");
    assertEquals(3, sec4.getCurrentMaxThreads());
    ScanExecutorConfig sec6 = tc.getScanExecutors().stream().filter(c -> c.name.equals("meta")).findFirst().get();
    assertEquals(3, sec6.maxThreads);
    String prefix = Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey();
    tc.set(prefix + "hulksmash.threads", "66");
    tc.set(prefix + "hulksmash.priority", "3");
    tc.set(prefix + "hulksmash.prioritizer", "com.foo.ScanPrioritizer");
    tc.set(prefix + "hulksmash.prioritizer.opts.k1", "v1");
    tc.set(prefix + "hulksmash.prioritizer.opts.k2", "v3");
    executors = tc.getScanExecutors();
    assertEquals(3, executors.size());
    ScanExecutorConfig sec7 = executors.stream().filter(c -> c.name.equals("hulksmash")).findFirst().get();
    assertEquals(66, sec7.maxThreads);
    assertEquals(3, sec7.priority.getAsInt());
    assertEquals("com.foo.ScanPrioritizer", sec7.prioritizerClass.get());
    assertEquals(Map.of("k1", "v1", "k2", "v3"), sec7.prioritizerOpts);
    tc.set(prefix + "hulksmash.threads", "44");
    assertEquals(66, sec7.maxThreads);
    assertEquals(44, sec7.getCurrentMaxThreads());
    ScanExecutorConfig sec8 = tc.getScanExecutors().stream().filter(c -> c.name.equals("hulksmash")).findFirst().get();
    assertEquals(44, sec8.maxThreads);
}

Example 2 with ScanExecutorConfig

use of org.apache.accumulo.core.conf.AccumuloConfiguration.ScanExecutorConfig in project accumulo by apache.

the class TabletServerResourceManager method createPriorityExecutor.

private ThreadPoolExecutor createPriorityExecutor(ScanExecutorConfig sec, Map<String, Queue<Runnable>> scanExecQueues) {
    BlockingQueue<Runnable> queue;
    if (sec.prioritizerClass.orElse("").isEmpty()) {
        queue = new LinkedBlockingQueue<>();
    } else {
        ScanPrioritizer factory = null;
        try {
            factory = ConfigurationTypeHelper.getClassInstance(null, sec.prioritizerClass.get(), ScanPrioritizer.class);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        if (factory == null) {
            queue = new LinkedBlockingQueue<>();
        } else {
            Comparator<ScanInfo> comparator = factory.createComparator(new ScanPrioritizer.CreateParameters() {

                private final ServiceEnvironment senv = new ServiceEnvironmentImpl(context);

                @Override
                public Map<String, String> getOptions() {
                    return sec.prioritizerOpts;
                }

                @Override
                public ServiceEnvironment getServiceEnv() {
                    return senv;
                }
            });
            // function to extract scan session from runnable
            Function<Runnable, ScanInfo> extractor = r -> ((ScanSession.ScanMeasurer) TraceUtil.unwrap(r)).getScanInfo();
            queue = new PriorityBlockingQueue<>(sec.maxThreads, Comparator.comparing(extractor, comparator));
        }
    }
    scanExecQueues.put(sec.name, queue);
    ThreadPoolExecutor es = ThreadPools.createThreadPool(sec.getCurrentMaxThreads(), sec.getCurrentMaxThreads(), 0L, TimeUnit.MILLISECONDS, "scan-" + sec.name, queue, sec.priority, true);
    modifyThreadPoolSizesAtRuntime(sec::getCurrentMaxThreads, "scan-" + sec.name, es);
    return es;
}