use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.
the class CloserThread method load.
//-------------------------------------------------------------------
// Initialization / Cleanup
//-------------------------------------------------------------------
/**
* Load the cores defined for this CoreContainer
*/
public void load() {
log.debug("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath());
// add the sharedLib to the shared resource loader before initializing cfg based plugins
String libDir = cfg.getSharedLibDirectory();
if (libDir != null) {
Path libPath = loader.getInstancePath().resolve(libDir);
try {
loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
loader.reloadLuceneSPI();
} catch (IOException e) {
if (!libDir.equals("lib")) {
// Don't complain if default "lib" dir does not exist
log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage());
}
}
}
metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig());
coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(coreContainerWorkExecutor, null, metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)), SolrMetricManager.mkName("coreContainerWorkExecutor", SolrInfoBean.Category.CONTAINER.toString(), "threadPool"));
shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
if (shardHandlerFactory instanceof SolrMetricProducer) {
SolrMetricProducer metricProducer = (SolrMetricProducer) shardHandlerFactory;
metricProducer.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), "httpShardHandler");
}
updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
updateShardHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), "updateShardHandler");
transientCoreCache = TransientSolrCoreCacheFactory.newInstance(loader, this);
logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
hostName = cfg.getNodeName();
zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig());
if (isZooKeeperAware())
pkiAuthenticationPlugin = new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName());
MDCLoggingContext.setNode(this);
securityConfHandler = isZooKeeperAware() ? new SecurityConfHandlerZk(this) : new SecurityConfHandlerLocal(this);
reloadSecurityProperties();
this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());
createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
metricsHandler = createHandler(METRICS_PATH, MetricsHandler.class.getName(), MetricsHandler.class);
metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);
// may want to add some configuration here in the future
metricsCollectorHandler.init(null);
containerHandlers.put(AUTHZ_PATH, securityConfHandler);
securityConfHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), AUTHZ_PATH);
containerHandlers.put(AUTHC_PATH, securityConfHandler);
if (pkiAuthenticationPlugin != null)
containerHandlers.put(PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler());
PluginInfo[] metricReporters = cfg.getMetricsConfig().getMetricReporters();
metricManager.loadReporters(metricReporters, loader, null, SolrInfoBean.Group.node);
metricManager.loadReporters(metricReporters, loader, null, SolrInfoBean.Group.jvm);
metricManager.loadReporters(metricReporters, loader, null, SolrInfoBean.Group.jetty);
coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController);
containerProperties.putAll(cfg.getSolrProperties());
// initialize gauges for reporting the number of cores and disk total/free
String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node);
metricManager.registerGauge(null, registryName, () -> solrCores.getCores().size(), true, "loaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(null, registryName, () -> solrCores.getLoadedCoreNames().size() - solrCores.getCores().size(), true, "lazy", SolrInfoBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(null, registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getLoadedCoreNames().size(), true, "unloaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toFile().getTotalSpace(), true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toFile().getUsableSpace(), true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
// add version information
metricManager.registerGauge(null, registryName, () -> this.getClass().getPackage().getSpecificationVersion(), true, "specification", SolrInfoBean.Category.CONTAINER.toString(), "version");
metricManager.registerGauge(null, registryName, () -> this.getClass().getPackage().getImplementationVersion(), true, "implementation", SolrInfoBean.Category.CONTAINER.toString(), "version");
SolrFieldCacheBean fieldCacheBean = new SolrFieldCacheBean();
fieldCacheBean.initializeMetrics(metricManager, registryName, null);
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this);
}
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService(ExecutorUtil.newMDCAwareFixedThreadPool(cfg.getCoreLoadThreadCount(isZooKeeperAware()), new DefaultSolrThreadFactory("coreLoadExecutor")), null, metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)), SolrMetricManager.mkName("coreLoadExecutor", SolrInfoBean.Category.CONTAINER.toString(), "threadPool"));
final List<Future<SolrCore>> futures = new ArrayList<>();
try {
List<CoreDescriptor> cds = coresLocator.discover(this);
if (isZooKeeperAware()) {
//sort the cores if it is in SolrCloud. In standalone node the order does not matter
CoreSorter coreComparator = new CoreSorter().init(this);
//make a copy
cds = new ArrayList<>(cds);
Collections.sort(cds, coreComparator::compare);
}
checkForDuplicateCoreNames(cds);
status |= CORE_DISCOVERY_COMPLETE;
for (final CoreDescriptor cd : cds) {
if (cd.isTransient() || !cd.isLoadOnStartup()) {
getTransientCacheHandler().addTransientDescriptor(cd.getName(), cd);
} else if (asyncSolrCoreLoad) {
solrCores.markCoreAsLoading(cd);
}
if (cd.isLoadOnStartup()) {
futures.add(coreLoadExecutor.submit(() -> {
SolrCore core;
try {
if (zkSys.getZkController() != null) {
zkSys.getZkController().throwErrorIfReplicaReplaced(cd);
}
core = create(cd, false, false);
} finally {
if (asyncSolrCoreLoad) {
solrCores.markCoreAsNotLoading(cd);
}
}
try {
zkSys.registerInZk(core, true, false);
} catch (RuntimeException e) {
SolrException.log(log, "Error registering SolrCore", e);
}
return core;
}));
}
}
// Start the background thread
backgroundCloser = new CloserThread(this, solrCores, cfg);
backgroundCloser.start();
} finally {
if (asyncSolrCoreLoad && futures != null) {
coreContainerWorkExecutor.submit((Runnable) () -> {
try {
for (Future<SolrCore> future : futures) {
try {
future.get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
log.error("Error waiting for SolrCore to be created", e);
}
}
} finally {
ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
}
});
} else {
ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
}
}
if (isZooKeeperAware()) {
zkSys.getZkController().checkOverseerDesignate();
}
// This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.
status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
}
use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.
the class SolrInfoBeanTest method testCallMBeanInfo.
/**
* Gets a list of everything we can find in the classpath and makes sure it has
* a name, description, etc...
*/
public void testCallMBeanInfo() throws Exception {
List<Class> classes = new ArrayList<>();
classes.addAll(getClassesForPackage(StandardRequestHandler.class.getPackage().getName()));
classes.addAll(getClassesForPackage(SearchComponent.class.getPackage().getName()));
classes.addAll(getClassesForPackage(LukeRequestHandler.class.getPackage().getName()));
classes.addAll(getClassesForPackage(DefaultSolrHighlighter.class.getPackage().getName()));
classes.addAll(getClassesForPackage(LRUCache.class.getPackage().getName()));
// System.out.println(classes);
int checked = 0;
SolrMetricManager metricManager = h.getCoreContainer().getMetricManager();
String registry = h.getCore().getCoreMetricManager().getRegistryName();
String scope = TestUtil.randomSimpleString(random(), 2, 10);
for (Class clazz : classes) {
if (SolrInfoBean.class.isAssignableFrom(clazz)) {
try {
SolrInfoBean info = (SolrInfoBean) clazz.newInstance();
if (info instanceof SolrMetricProducer) {
((SolrMetricProducer) info).initializeMetrics(metricManager, registry, scope);
}
//System.out.println( info.getClass() );
assertNotNull(info.getName());
assertNotNull(info.getDescription());
assertNotNull(info.getCategory());
if (info instanceof LRUCache) {
continue;
}
assertNotNull(info.toString());
checked++;
} catch (InstantiationException ex) {
// expected...
//System.out.println( "unable to initialize: "+clazz );
}
}
}
assertTrue("there are at least 10 SolrInfoBean that should be found in the classpath, found " + checked, checked > 10);
}
use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.
the class SolrJmxReporterTest method testReportMetrics.
@Test
public void testReportMetrics() throws Exception {
Random random = random();
Map<String, Counter> registered = new HashMap<>();
String scope = SolrMetricTestUtils.getRandomScope(random, true);
SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random, true);
int iterations = TestUtil.nextInt(random, 0, MAX_ITERATIONS);
for (int i = 0; i < iterations; ++i) {
Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetricsWithReplacements(random, registered);
SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
coreMetricManager.registerMetricProducer(scope, producer);
registered.putAll(metrics);
//waitForListener();
Set<ObjectInstance> objects = mBeanServer.queryMBeans(null, null);
assertEquals(registered.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && rootName.equals(o.getObjectName().getDomain())).count());
}
}
use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.
the class SolrJmxReporterTest method testEnabled.
@Test
public void testEnabled() throws Exception {
String root1 = TestUtil.randomSimpleString(random(), 5, 10);
PluginInfo pluginInfo1 = createReporterPluginInfo(root1, true);
metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo1, coreMetricManager.getTag());
String root2 = TestUtil.randomSimpleString(random(), 5, 10);
assertFalse(root2.equals(root1));
PluginInfo pluginInfo2 = createReporterPluginInfo(root2, false);
metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo2, coreMetricManager.getTag());
Map<String, SolrMetricReporter> reporters = metricManager.getReporters(coreMetricManager.getRegistryName());
assertTrue(reporters.containsKey(pluginInfo1.name + "@" + coreMetricManager.getTag()));
assertTrue(reporters.containsKey(pluginInfo2.name + "@" + coreMetricManager.getTag()));
String scope = SolrMetricTestUtils.getRandomScope(random(), true);
SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random(), true);
Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetrics(random(), true);
SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
coreMetricManager.registerMetricProducer(scope, producer);
Set<ObjectInstance> objects = mBeanServer.queryMBeans(null, null);
assertEquals(metrics.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && root1.equals(o.getObjectName().getDomain())).count());
assertEquals(0, objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && root2.equals(o.getObjectName().getDomain())).count());
}
use of org.apache.solr.metrics.SolrMetricProducer in project lucene-solr by apache.
the class SolrJmxReporterTest method testReloadCore.
@Test
public void testReloadCore() throws Exception {
Random random = random();
String scope = SolrMetricTestUtils.getRandomScope(random, true);
SolrInfoBean.Category category = SolrMetricTestUtils.getRandomCategory(random, true);
Map<String, Counter> metrics = SolrMetricTestUtils.getRandomMetrics(random, true);
SolrMetricProducer producer = SolrMetricTestUtils.getProducerOf(metricManager, category, scope, metrics);
coreMetricManager.registerMetricProducer(scope, producer);
Set<ObjectInstance> objects = mBeanServer.queryMBeans(null, null);
assertEquals(metrics.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && o.getObjectName().getDomain().equals(rootName)).count());
h.getCoreContainer().reload(h.getCore().getName());
PluginInfo pluginInfo = createReporterPluginInfo(rootName, true);
metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo, String.valueOf(coreMetricManager.getCore().hashCode()));
coreMetricManager.registerMetricProducer(scope, producer);
objects = mBeanServer.queryMBeans(null, null);
assertEquals(metrics.size(), objects.stream().filter(o -> scope.equals(o.getObjectName().getKeyProperty("scope")) && rootName.equals(o.getObjectName().getDomain())).count());
}
Aggregations