Examples with CoreContainer org.apache.solr.core.CoreContainer used on opensource projects

Example 41 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class ChaosMonkey method expireSession.

// TODO: expire all clients at once?
public void expireSession(final JettySolrRunner jetty) {
    CoreContainer cores = jetty.getCoreContainer();
    if (cores != null) {
        monkeyLog("expire session for " + jetty.getLocalPort() + " !");
        causeConnectionLoss(jetty);
        long sessionId = cores.getZkController().getZkClient().getSolrZooKeeper().getSessionId();
        zkServer.expire(sessionId);
    }
}

Example 42 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class ChaosMonkey method getRandomJetty.

public CloudJettyRunner getRandomJetty(String slice, boolean aggressivelyKillLeaders) throws KeeperException, InterruptedException {
    int numActive = 0;
    numActive = checkIfKillIsLegal(slice, numActive);
    // TODO: stale state makes this a tough call
    if (numActive < 2) {
        // we cannot kill anyone
        monkeyLog("only one active node in shard - monkey cannot kill :(");
        return null;
    }
    // let's check the deadpool count
    int numRunning = 0;
    for (CloudJettyRunner cjetty : shardToJetty.get(slice)) {
        if (!deadPool.contains(cjetty)) {
            numRunning++;
        }
    }
    if (numRunning < 2) {
        // we cannot kill anyone
        monkeyLog("only one active node in shard - monkey cannot kill :(");
        return null;
    }
    boolean canKillIndexer = canKillIndexer(slice);
    if (!canKillIndexer) {
        monkeyLog("Number of indexer nodes (nrt or tlog replicas) is not enough to kill one of them, Will only choose a pull replica to kill");
    }
    int chance = chaosRandom.nextInt(10);
    CloudJettyRunner cjetty = null;
    if (chance <= 5 && aggressivelyKillLeaders && canKillIndexer) {
        // if killLeader, really aggressively go after leaders
        cjetty = shardToLeaderJetty.get(slice);
    } else {
        List<CloudJettyRunner> jetties = shardToJetty.get(slice);
        // get random node
        int attempt = 0;
        while (true) {
            attempt++;
            int index = chaosRandom.nextInt(jetties.size());
            cjetty = jetties.get(index);
            if (canKillIndexer || getTypeForJetty(slice, cjetty) == Replica.Type.PULL) {
                break;
            } else if (attempt > 20) {
                monkeyLog("Can't kill indexer nodes (nrt or tlog replicas) and couldn't find a random pull node after 20 attempts - monkey cannot kill :(");
                return null;
            }
        }
        ZkNodeProps leader = null;
        try {
            leader = zkStateReader.getLeaderRetry(collection, slice);
        } catch (Throwable t) {
            log.error("Could not get leader", t);
            return null;
        }
        // cluster state can be stale - also go by our 'near real-time' is leader prop
        boolean rtIsLeader;
        CoreContainer cc = cjetty.jetty.getCoreContainer();
        if (cc != null) {
            try (SolrCore core = cc.getCore(leader.getStr(ZkStateReader.CORE_NAME_PROP))) {
                if (core == null) {
                    monkeyLog("selected jetty not running correctly - skip");
                    return null;
                }
                rtIsLeader = core.getCoreDescriptor().getCloudDescriptor().isLeader();
            }
        } else {
            return null;
        }
        boolean isLeader = leader.getStr(ZkStateReader.NODE_NAME_PROP).equals(cjetty.nodeName) || rtIsLeader;
        if (!aggressivelyKillLeaders && isLeader) {
            // we don't kill leaders...
            monkeyLog("abort! I don't kill leaders");
            return null;
        }
    }
    if (cjetty.jetty.getLocalPort() == -1) {
        // we can't kill the dead
        monkeyLog("abort! This guy is already dead");
        return null;
    }
    //System.out.println("num active:" + numActive + " for " + slice + " sac:" + jetty.getLocalPort());
    monkeyLog("chose a victim! " + cjetty.jetty.getLocalPort());
    return cjetty;
}

Example 43 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class TestTrackingShardHandlerFactory method testRequestTracking.

@Test
@BaseDistributedSearchTestCase.ShardsFixed(num = 2)
public void testRequestTracking() throws Exception {
    String collectionName = "testTwoPhase";
    List<JettySolrRunner> runners = new ArrayList<>(jettys);
    runners.add(controlJetty);
    TrackingShardHandlerFactory.RequestTrackingQueue trackingQueue = new TrackingShardHandlerFactory.RequestTrackingQueue();
    TrackingShardHandlerFactory.setTrackingQueue(runners, trackingQueue);
    for (JettySolrRunner runner : runners) {
        CoreContainer container = runner.getCoreContainer();
        ShardHandlerFactory factory = container.getShardHandlerFactory();
        assert factory instanceof TrackingShardHandlerFactory;
        TrackingShardHandlerFactory trackingShardHandlerFactory = (TrackingShardHandlerFactory) factory;
        assertSame(trackingQueue, trackingShardHandlerFactory.getTrackingQueue());
    }
    createCollection(collectionName, 2, 1, 1);
    waitForRecoveriesToFinish(collectionName, true);
    List<TrackingShardHandlerFactory.ShardRequestAndParams> coreAdminRequests = trackingQueue.getCoreAdminRequests();
    assertNotNull(coreAdminRequests);
    assertEquals("Unexpected number of core admin requests were found", 2, coreAdminRequests.size());
    CloudSolrClient client = cloudClient;
    client.setDefaultCollection(collectionName);
    /*
        hash of b is 95de7e03 high bits=2 shard=shard1
        hash of e is 656c4367 high bits=1 shard=shard2
         */
    for (int i = 0; i < 10; i++) {
        SolrInputDocument doc = new SolrInputDocument();
        doc.addField("id", (i % 2 == 0 ? "b!" : "e!") + i);
        doc.addField("a_i", i);
        doc.addField("a_t", "text_" + i);
        client.add(doc);
    }
    client.commit();
    client.query(new SolrQuery("*:*"));
    TrackingShardHandlerFactory.ShardRequestAndParams getTopIdsRequest = trackingQueue.getShardRequestByPurpose(client.getZkStateReader(), collectionName, "shard1", ShardRequest.PURPOSE_GET_TOP_IDS);
    assertNotNull(getTopIdsRequest);
    getTopIdsRequest = trackingQueue.getShardRequestByPurpose(client.getZkStateReader(), collectionName, "shard2", ShardRequest.PURPOSE_GET_TOP_IDS);
    assertNotNull(getTopIdsRequest);
    TrackingShardHandlerFactory.ShardRequestAndParams getFieldsRequest = trackingQueue.getShardRequestByPurpose(client.getZkStateReader(), collectionName, "shard1", ShardRequest.PURPOSE_GET_FIELDS);
    assertNotNull(getFieldsRequest);
    getFieldsRequest = trackingQueue.getShardRequestByPurpose(client.getZkStateReader(), collectionName, "shard2", ShardRequest.PURPOSE_GET_FIELDS);
    assertNotNull(getFieldsRequest);
    int numRequests = 0;
    Map<String, List<TrackingShardHandlerFactory.ShardRequestAndParams>> allRequests = trackingQueue.getAllRequests();
    for (Map.Entry<String, List<TrackingShardHandlerFactory.ShardRequestAndParams>> entry : allRequests.entrySet()) {
        numRequests += entry.getValue().size();
    }
    // 4 shard requests + 2 core admin requests (invoked by create collection API)
    assertEquals("Total number of requests do not match expected", 6, numRequests);
    // reset
    TrackingShardHandlerFactory.setTrackingQueue(runners, null);
    for (JettySolrRunner runner : runners) {
        CoreContainer container = runner.getCoreContainer();
        ShardHandlerFactory factory = container.getShardHandlerFactory();
        assert factory instanceof TrackingShardHandlerFactory;
        TrackingShardHandlerFactory trackingShardHandlerFactory = (TrackingShardHandlerFactory) factory;
        assertFalse(trackingShardHandlerFactory.isTracking());
    }
    // make another request and verify
    client.query(new SolrQuery("*:*"));
    numRequests = 0;
    allRequests = trackingQueue.getAllRequests();
    for (Map.Entry<String, List<TrackingShardHandlerFactory.ShardRequestAndParams>> entry : allRequests.entrySet()) {
        numRequests += entry.getValue().size();
    }
    // should still be 6
    assertEquals("Total number of shard requests do not match expected", 6, numRequests);
}

Example 44 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class SolrCloudReportersTest method testExplicitConfiguration.

@Test
public void testExplicitConfiguration() throws Exception {
    String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr-solrreporter.xml"), "UTF-8");
    configureCluster(2).withSolrXml(solrXml).configure();
    cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test");
    System.out.println("ZK: " + cluster.getZkServer().getZkAddress());
    CollectionAdminRequest.createCollection("test_collection", "test", 2, 2).setMaxShardsPerNode(4).process(cluster.getSolrClient());
    waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2));
    Thread.sleep(15000);
    cluster.getJettySolrRunners().forEach(jetty -> {
        CoreContainer cc = jetty.getCoreContainer();
        for (String name : cc.getLoadedCoreNames()) {
            SolrCore core = cc.getCore(name);
            try {
                String registryName = core.getCoreMetricManager().getRegistryName();
                String leaderRegistryName = core.getCoreMetricManager().getLeaderRegistryName();
                String coreName = core.getName();
                String collectionName = core.getCoreDescriptor().getCollectionName();
                String coreNodeName = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
                String replicaName = coreName.substring(coreName.indexOf("_replica_") + 1);
                String shardId = core.getCoreDescriptor().getCloudDescriptor().getShardId();
                assertEquals("solr.core." + collectionName + "." + shardId + "." + replicaName, registryName);
                assertEquals("solr.collection." + collectionName + "." + shardId + ".leader", leaderRegistryName);
            } finally {
                if (core != null) {
                    core.close();
                }
            }
        }
        SolrMetricManager metricManager = cc.getMetricManager();
        Map<String, SolrMetricReporter> reporters = metricManager.getReporters("solr.cluster");
        assertEquals(reporters.toString(), 1, reporters.size());
        SolrMetricReporter reporter = reporters.get("test");
        assertNotNull(reporter);
        assertTrue(reporter.toString(), reporter instanceof SolrClusterReporter);
        SolrClusterReporter sor = (SolrClusterReporter) reporter;
        assertEquals(5, sor.getPeriod());
        for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.replica.*")) {
            reporters = metricManager.getReporters(registryName);
            assertEquals(reporters.toString(), 1 + jmxReporter, reporters.size());
            reporter = null;
            for (String name : reporters.keySet()) {
                if (name.startsWith("test")) {
                    reporter = reporters.get(name);
                }
            }
            assertNotNull(reporter);
            assertTrue(reporter.toString(), reporter instanceof SolrShardReporter);
            SolrShardReporter srr = (SolrShardReporter) reporter;
            assertEquals(5, srr.getPeriod());
        }
        for (String registryName : metricManager.registryNames(".*\\.leader")) {
            leaderRegistries++;
            reporters = metricManager.getReporters(registryName);
            assertEquals(reporters.toString(), 0, reporters.size());
            Map<String, Metric> metrics = metricManager.registry(registryName).getMetrics();
            String key = "QUERY./select.requests";
            assertTrue(key, metrics.containsKey(key));
            assertTrue(key, metrics.get(key) instanceof AggregateMetric);
            key = "UPDATE./update/json.requests";
            assertTrue(key, metrics.containsKey(key));
            assertTrue(key, metrics.get(key) instanceof AggregateMetric);
        }
        if (metricManager.registryNames().contains("solr.cluster")) {
            clusterRegistries++;
            Map<String, Metric> metrics = metricManager.registry("solr.cluster").getMetrics();
            String key = "jvm.memory.heap.init";
            assertTrue(key, metrics.containsKey(key));
            assertTrue(key, metrics.get(key) instanceof AggregateMetric);
            key = "leader.test_collection.shard1.UPDATE./update/json.requests.max";
            assertTrue(key, metrics.containsKey(key));
            assertTrue(key, metrics.get(key) instanceof AggregateMetric);
        }
    });
    assertEquals("leaderRegistries", 2, leaderRegistries);
    assertEquals("clusterRegistries", 1, clusterRegistries);
}

Example 45 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class SolrShardReporterTest method test.

@Test
public void test() throws Exception {
    waitForRecoveriesToFinish("control_collection", jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false);
    waitForRecoveriesToFinish("collection1", jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false);
    printLayout();
    // wait for at least two reports
    Thread.sleep(10000);
    ClusterState state = jettys.get(0).getCoreContainer().getZkController().getClusterState();
    for (JettySolrRunner jetty : jettys) {
        CoreContainer cc = jetty.getCoreContainer();
        SolrMetricManager metricManager = cc.getMetricManager();
        for (final String coreName : cc.getLoadedCoreNames()) {
            CoreDescriptor cd = cc.getCoreDescriptor(coreName);
            if (cd.getCloudDescriptor() == null) {
                // not a cloud collection
                continue;
            }
            CloudDescriptor cloudDesc = cd.getCloudDescriptor();
            DocCollection docCollection = state.getCollection(cloudDesc.getCollectionName());
            String replicaName = SolrCoreMetricManager.parseReplicaName(cloudDesc.getCollectionName(), coreName);
            if (replicaName == null) {
                replicaName = cloudDesc.getCoreNodeName();
            }
            String registryName = SolrCoreMetricManager.createRegistryName(true, cloudDesc.getCollectionName(), cloudDesc.getShardId(), replicaName, null);
            String leaderRegistryName = SolrCoreMetricManager.createLeaderRegistryName(true, cloudDesc.getCollectionName(), cloudDesc.getShardId());
            boolean leader = cloudDesc.isLeader();
            Slice slice = docCollection.getSlice(cloudDesc.getShardId());
            int numReplicas = slice.getReplicas().size();
            if (leader) {
                assertTrue(metricManager.registryNames() + " doesn't contain " + leaderRegistryName, metricManager.registryNames().contains(leaderRegistryName));
                Map<String, Metric> metrics = metricManager.registry(leaderRegistryName).getMetrics();
                metrics.forEach((k, v) -> {
                    assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v, v instanceof AggregateMetric);
                    AggregateMetric am = (AggregateMetric) v;
                    if (!k.startsWith("REPLICATION.peerSync")) {
                        assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), numReplicas, am.size());
                    }
                });
            } else {
                assertFalse(metricManager.registryNames() + " contains " + leaderRegistryName + " but it's not a leader!", metricManager.registryNames().contains(leaderRegistryName));
                Map<String, Metric> metrics = metricManager.registry(leaderRegistryName).getMetrics();
                metrics.forEach((k, v) -> {
                    assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v, v instanceof AggregateMetric);
                    AggregateMetric am = (AggregateMetric) v;
                    if (!k.startsWith("REPLICATION.peerSync")) {
                        assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), 1, am.size());
                    }
                });
            }
            assertTrue(metricManager.registryNames() + " doesn't contain " + registryName, metricManager.registryNames().contains(registryName));
        }
    }
    SolrMetricManager metricManager = controlJetty.getCoreContainer().getMetricManager();
    assertTrue(metricManager.registryNames().contains("solr.cluster"));
}