Search in sources :

Example 1 with SolrMetricProducer

use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.

the class CloserThread method load.

//-------------------------------------------------------------------
// Initialization / Cleanup
//-------------------------------------------------------------------
/**
   * Load the cores defined for this CoreContainer
   */
public void load() {
    log.debug("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath());
    // add the sharedLib to the shared resource loader before initializing cfg based plugins
    String libDir = cfg.getSharedLibDirectory();
    if (libDir != null) {
        Path libPath = loader.getInstancePath().resolve(libDir);
        try {
            loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
            loader.reloadLuceneSPI();
        } catch (IOException e) {
            if (!libDir.equals("lib")) {
                // Don't complain if default "lib" dir does not exist
                log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage());
            }
        }
    }
    metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig());
    coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(coreContainerWorkExecutor, null, metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)), SolrMetricManager.mkName("coreContainerWorkExecutor", SolrInfoBean.Category.CONTAINER.toString(), "threadPool"));
    shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
    if (shardHandlerFactory instanceof SolrMetricProducer) {
        SolrMetricProducer metricProducer = (SolrMetricProducer) shardHandlerFactory;
        metricProducer.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), "httpShardHandler");
    }
    updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
    updateShardHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), "updateShardHandler");
    transientCoreCache = TransientSolrCoreCacheFactory.newInstance(loader, this);
    logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
    hostName = cfg.getNodeName();
    zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig());
    if (isZooKeeperAware())
        pkiAuthenticationPlugin = new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName());
    MDCLoggingContext.setNode(this);
    securityConfHandler = isZooKeeperAware() ? new SecurityConfHandlerZk(this) : new SecurityConfHandlerLocal(this);
    reloadSecurityProperties();
    this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());
    createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
    collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
    infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
    coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
    configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
    metricsHandler = createHandler(METRICS_PATH, MetricsHandler.class.getName(), MetricsHandler.class);
    metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);
    // may want to add some configuration here in the future
    metricsCollectorHandler.init(null);
    containerHandlers.put(AUTHZ_PATH, securityConfHandler);
    securityConfHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), AUTHZ_PATH);
    containerHandlers.put(AUTHC_PATH, securityConfHandler);
    if (pkiAuthenticationPlugin != null)
        containerHandlers.put(PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler());
    PluginInfo[] metricReporters = cfg.getMetricsConfig().getMetricReporters();
    metricManager.loadReporters(metricReporters, loader, null, SolrInfoBean.Group.node);
    metricManager.loadReporters(metricReporters, loader, null, SolrInfoBean.Group.jvm);
    metricManager.loadReporters(metricReporters, loader, null, SolrInfoBean.Group.jetty);
    coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController);
    containerProperties.putAll(cfg.getSolrProperties());
    // initialize gauges for reporting the number of cores and disk total/free
    String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node);
    metricManager.registerGauge(null, registryName, () -> solrCores.getCores().size(), true, "loaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
    metricManager.registerGauge(null, registryName, () -> solrCores.getLoadedCoreNames().size() - solrCores.getCores().size(), true, "lazy", SolrInfoBean.Category.CONTAINER.toString(), "cores");
    metricManager.registerGauge(null, registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getLoadedCoreNames().size(), true, "unloaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
    metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toFile().getTotalSpace(), true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
    metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toFile().getUsableSpace(), true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
    // add version information
    metricManager.registerGauge(null, registryName, () -> this.getClass().getPackage().getSpecificationVersion(), true, "specification", SolrInfoBean.Category.CONTAINER.toString(), "version");
    metricManager.registerGauge(null, registryName, () -> this.getClass().getPackage().getImplementationVersion(), true, "implementation", SolrInfoBean.Category.CONTAINER.toString(), "version");
    SolrFieldCacheBean fieldCacheBean = new SolrFieldCacheBean();
    fieldCacheBean.initializeMetrics(metricManager, registryName, null);
    if (isZooKeeperAware()) {
        metricManager.loadClusterReporters(metricReporters, this);
    }
    // setup executor to load cores in parallel
    ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(ExecutorUtil.newMDCAwareFixedThreadPool(cfg.getCoreLoadThreadCount(isZooKeeperAware()), new DefaultSolrThreadFactory("coreLoadExecutor")), null, metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)), SolrMetricManager.mkName("coreLoadExecutor", SolrInfoBean.Category.CONTAINER.toString(), "threadPool"));
    final List<Future<SolrCore>> futures = new ArrayList<>();
    try {
        List<CoreDescriptor> cds = coresLocator.discover(this);
        if (isZooKeeperAware()) {
            //sort the cores if it is in SolrCloud. In standalone node the order does not matter
            CoreSorter coreComparator = new CoreSorter().init(this);
            //make a copy
            cds = new ArrayList<>(cds);
            Collections.sort(cds, coreComparator::compare);
        }
        checkForDuplicateCoreNames(cds);
        status |= CORE_DISCOVERY_COMPLETE;
        for (final CoreDescriptor cd : cds) {
            if (cd.isTransient() || !cd.isLoadOnStartup()) {
                getTransientCacheHandler().addTransientDescriptor(cd.getName(), cd);
            } else if (asyncSolrCoreLoad) {
                solrCores.markCoreAsLoading(cd);
            }
            if (cd.isLoadOnStartup()) {
                futures.add(coreLoadExecutor.submit(() -> {
                    SolrCore core;
                    try {
                        if (zkSys.getZkController() != null) {
                            zkSys.getZkController().throwErrorIfReplicaReplaced(cd);
                        }
                        core = create(cd, false, false);
                    } finally {
                        if (asyncSolrCoreLoad) {
                            solrCores.markCoreAsNotLoading(cd);
                        }
                    }
                    try {
                        zkSys.registerInZk(core, true, false);
                    } catch (RuntimeException e) {
                        SolrException.log(log, "Error registering SolrCore", e);
                    }
                    return core;
                }));
            }
        }
        // Start the background thread
        backgroundCloser = new CloserThread(this, solrCores, cfg);
        backgroundCloser.start();
    } finally {
        if (asyncSolrCoreLoad && futures != null) {
            coreContainerWorkExecutor.submit((Runnable) () -> {
                try {
                    for (Future<SolrCore> future : futures) {
                        try {
                            future.get();
                        } catch (InterruptedException e) {
                            Thread.currentThread().interrupt();
                        } catch (ExecutionException e) {
                            log.error("Error waiting for SolrCore to be created", e);
                        }
                    }
                } finally {
                    ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
                }
            });
        } else {
            ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
        }
    }
    if (isZooKeeperAware()) {
        zkSys.getZkController().checkOverseerDesignate();
    }
    // This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.
    status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
}
Also used : PKIAuthenticationPlugin(org.apache.solr.security.PKIAuthenticationPlugin) SecurityConfHandlerLocal(org.apache.solr.handler.admin.SecurityConfHandlerLocal) CollectionsHandler(org.apache.solr.handler.admin.CollectionsHandler) SolrMetricProducer(org.apache.solr.metrics.SolrMetricProducer) CoreAdminHandler(org.apache.solr.handler.admin.CoreAdminHandler) ArrayList(java.util.ArrayList) UpdateShardHandler(org.apache.solr.update.UpdateShardHandler) SecurityConfHandlerZk(org.apache.solr.handler.admin.SecurityConfHandlerZk) BackupRepositoryFactory(org.apache.solr.core.backup.repository.BackupRepositoryFactory) MetricsCollectorHandler(org.apache.solr.handler.admin.MetricsCollectorHandler) MetricsHandler(org.apache.solr.handler.admin.MetricsHandler) SolrMetricManager(org.apache.solr.metrics.SolrMetricManager) ZookeeperInfoHandler(org.apache.solr.handler.admin.ZookeeperInfoHandler) SolrFieldCacheBean(org.apache.solr.search.SolrFieldCacheBean) ExecutionException(java.util.concurrent.ExecutionException) Path(java.nio.file.Path) ConfigSetsHandler(org.apache.solr.handler.admin.ConfigSetsHandler) DefaultSolrThreadFactory(org.apache.solr.util.DefaultSolrThreadFactory) IOException(java.io.IOException) ZookeeperInfoHandler(org.apache.solr.handler.admin.ZookeeperInfoHandler) InfoHandler(org.apache.solr.handler.admin.InfoHandler) ExecutorService(java.util.concurrent.ExecutorService) Future(java.util.concurrent.Future)

Example 2 with SolrMetricProducer

use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.

the class SolrInfoBeanTest method testCallMBeanInfo.

/**
   * Gets a list of everything we can find in the classpath and makes sure it has
   * a name, description, etc...
   */
public void testCallMBeanInfo() throws Exception {
    List<Class> classes = new ArrayList<>();
    classes.addAll(getClassesForPackage(StandardRequestHandler.class.getPackage().getName()));
    classes.addAll(getClassesForPackage(SearchComponent.class.getPackage().getName()));
    classes.addAll(getClassesForPackage(LukeRequestHandler.class.getPackage().getName()));
    classes.addAll(getClassesForPackage(DefaultSolrHighlighter.class.getPackage().getName()));
    classes.addAll(getClassesForPackage(LRUCache.class.getPackage().getName()));
    // System.out.println(classes);
    int checked = 0;
    SolrMetricManager metricManager = h.getCoreContainer().getMetricManager();
    String registry = h.getCore().getCoreMetricManager().getRegistryName();
    String scope = TestUtil.randomSimpleString(random(), 2, 10);
    for (Class clazz : classes) {
        if (SolrInfoBean.class.isAssignableFrom(clazz)) {
            try {
                SolrInfoBean info = (SolrInfoBean) clazz.newInstance();
                if (info instanceof SolrMetricProducer) {
                    ((SolrMetricProducer) info).initializeMetrics(metricManager, registry, scope);
                }
                //System.out.println( info.getClass() );
                assertNotNull(info.getName());
                assertNotNull(info.getDescription());
                assertNotNull(info.getCategory());
                if (info instanceof LRUCache) {
                    continue;
                }
                assertNotNull(info.toString());
                checked++;
            } catch (InstantiationException ex) {
            // expected...
            //System.out.println( "unable to initialize: "+clazz );
            }
        }
    }
    assertTrue("there are at least 10 SolrInfoBean that should be found in the classpath, found " + checked, checked > 10);
}
Also used : SolrInfoBean(org.apache.solr.core.SolrInfoBean) ArrayList(java.util.ArrayList) SolrMetricProducer(org.apache.solr.metrics.SolrMetricProducer) LRUCache(org.apache.solr.search.LRUCache) LukeRequestHandler(org.apache.solr.handler.admin.LukeRequestHandler) SearchComponent(org.apache.solr.handler.component.SearchComponent) DefaultSolrHighlighter(org.apache.solr.highlight.DefaultSolrHighlighter) SolrMetricManager(org.apache.solr.metrics.SolrMetricManager) BeforeClass(org.junit.BeforeClass) StandardRequestHandler(org.apache.solr.handler.StandardRequestHandler)

Example 3 with SolrMetricProducer

use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.

the class SolrJmxReporterTest method testReportMetrics.

@Test
public void testReportMetrics() throws Exception {
    Random random = random();
    Map<String, Counter> registered = new HashMap<>();
    String scope = SolrMetricTestUtils.getRandomScope(random, true);
    SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random, true);
    int iterations = TestUtil.nextInt(random, 0, MAX_ITERATIONS);
    for (int i = 0; i < iterations; ++i) {
        Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetricsWithReplacements(random, registered);
        SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
        coreMetricManager.registerMetricProducer(scope, producer);
        registered.putAll(metrics);
        //waitForListener();
        Set<ObjectInstance> objects = mBeanServer.queryMBeans(null, null);
        assertEquals(registered.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && rootName.equals(o.getObjectName().getDomain())).count());
    }
}
Also used : Counter(com.codahale.metrics.Counter) Random(java.util.Random) SolrInfoBean(org.apache.solr.core.SolrInfoBean) HashMap(java.util.HashMap) SolrMetricProducer(org.apache.solr.metrics.SolrMetricProducer) ObjectInstance(javax.management.ObjectInstance) Test(org.junit.Test)

Example 4 with SolrMetricProducer

use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.

the class SolrJmxReporterTest method testEnabled.

@Test
public void testEnabled() throws Exception {
    String root1 = TestUtil.randomSimpleString(random(), 5, 10);
    PluginInfo pluginInfo1 = createReporterPluginInfo(root1, true);
    metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo1, coreMetricManager.getTag());
    String root2 = TestUtil.randomSimpleString(random(), 5, 10);
    assertFalse(root2.equals(root1));
    PluginInfo pluginInfo2 = createReporterPluginInfo(root2, false);
    metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo2, coreMetricManager.getTag());
    Map<String, SolrMetricReporter> reporters = metricManager.getReporters(coreMetricManager.getRegistryName());
    assertTrue(reporters.containsKey(pluginInfo1.name + "@" + coreMetricManager.getTag()));
    assertTrue(reporters.containsKey(pluginInfo2.name + "@" + coreMetricManager.getTag()));
    String scope = SolrMetricTestUtils.getRandomScope(random(), true);
    SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random(), true);
    Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetrics(random(), true);
    SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
    coreMetricManager.registerMetricProducer(scope, producer);
    Set<ObjectInstance> objects = mBeanServer.queryMBeans(null, null);
    assertEquals(metrics.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && root1.equals(o.getObjectName().getDomain())).count());
    assertEquals(0, objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && root2.equals(o.getObjectName().getDomain())).count());
}
Also used : Counter(com.codahale.metrics.Counter) SolrInfoBean(org.apache.solr.core.SolrInfoBean) SolrMetricProducer(org.apache.solr.metrics.SolrMetricProducer) ObjectInstance(javax.management.ObjectInstance) PluginInfo(org.apache.solr.core.PluginInfo) SolrMetricReporter(org.apache.solr.metrics.SolrMetricReporter) Test(org.junit.Test)

Example 5 with SolrMetricProducer

use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.

the class SolrJmxReporterTest method testReloadCore.

@Test
public void testReloadCore() throws Exception {
    Random random = random();
    String scope = SolrMetricTestUtils.getRandomScope(random, true);
    SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random, true);
    Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetrics(random, true);
    SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
    coreMetricManager.registerMetricProducer(scope, producer);
    Set<ObjectInstance> objects = mBeanServer.queryMBeans(null, null);
    assertEquals(metrics.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && o.getObjectName().getDomain().equals(rootName)).count());
    h.getCoreContainer().reload(h.getCore().getName());
    PluginInfo pluginInfo = createReporterPluginInfo(rootName, true);
    metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo, String.valueOf(coreMetricManager.getCore().hashCode()));
    coreMetricManager.registerMetricProducer(scope, producer);
    objects = mBeanServer.queryMBeans(null, null);
    assertEquals(metrics.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && rootName.equals(o.getObjectName().getDomain())).count());
}
Also used : Counter(com.codahale.metrics.Counter) Random(java.util.Random) SolrInfoBean(org.apache.solr.core.SolrInfoBean) SolrMetricProducer(org.apache.solr.metrics.SolrMetricProducer) ObjectInstance(javax.management.ObjectInstance) PluginInfo(org.apache.solr.core.PluginInfo) Test(org.junit.Test)

Aggregations

SolrMetricProducer (org.apache.solr.metrics.SolrMetricProducer)7 SolrInfoBean (org.apache.solr.core.SolrInfoBean)4 Counter (com.codahale.metrics.Counter)3 ObjectInstance (javax.management.ObjectInstance)3 Test (org.junit.Test)3 ArrayList (java.util.ArrayList)2 Random (java.util.Random)2 PluginInfo (org.apache.solr.core.PluginInfo)2 SolrMetricManager (org.apache.solr.metrics.SolrMetricManager)2 IOException (java.io.IOException)1 Path (java.nio.file.Path)1 HashMap (java.util.HashMap)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 BackupRepositoryFactory (org.apache.solr.core.backup.repository.BackupRepositoryFactory)1 StandardRequestHandler (org.apache.solr.handler.StandardRequestHandler)1 CollectionsHandler (org.apache.solr.handler.admin.CollectionsHandler)1 ConfigSetsHandler (org.apache.solr.handler.admin.ConfigSetsHandler)1 CoreAdminHandler (org.apache.solr.handler.admin.CoreAdminHandler)1