Examples with ZkNodeProps org.apache.solr.common.cloud.ZkNodeProps used on opensource projects

Example 56 with ZkNodeProps

use of org.apache.solr.common.cloud.ZkNodeProps in project lucene-solr by apache.

the class ClusterStateUpdateTest method testCoreRegistration.

@Test
public void testCoreRegistration() throws Exception {
    System.setProperty("solrcloud.update.delay", "1");
    Map<String, Object> props2 = new HashMap<>();
    props2.put("configName", "conf1");
    ZkNodeProps zkProps2 = new ZkNodeProps(props2);
    SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
    zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/testcore", Utils.toJSON(zkProps2), CreateMode.PERSISTENT, true);
    zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/testcore/shards", CreateMode.PERSISTENT, true);
    zkClient.close();
    container1.create("testcore", ImmutableMap.of("dataDir", dataDir4.getAbsolutePath()));
    ZkController zkController2 = container2.getZkController();
    String host = zkController2.getHostName();
    // slight pause - TODO: takes an oddly long amount of time to schedule tasks
    // with almost no delay ...
    ClusterState clusterState2 = null;
    Map<String, Slice> slices = null;
    for (int i = 75; i > 0; i--) {
        clusterState2 = zkController2.getClusterState();
        slices = clusterState2.getSlicesMap("testcore");
        if (slices != null && slices.containsKey("shard1") && slices.get("shard1").getReplicasMap().size() > 0) {
            break;
        }
        Thread.sleep(500);
    }
    assertNotNull(slices);
    assertTrue(slices.containsKey("shard1"));
    Slice slice = slices.get("shard1");
    assertEquals("shard1", slice.getName());
    Map<String, Replica> shards = slice.getReplicasMap();
    assertEquals(1, shards.size());
    Replica zkProps = shards.get(host + ":1661_solr_testcore");
    assertNotNull(zkProps);
    assertEquals(host + ":1661_solr", zkProps.getStr(ZkStateReader.NODE_NAME_PROP));
    assertEquals("http://" + host + ":1661/solr", zkProps.getStr(ZkStateReader.BASE_URL_PROP));
    Set<String> liveNodes = clusterState2.getLiveNodes();
    assertNotNull(liveNodes);
    assertEquals(3, liveNodes.size());
    container3.shutdown();
    // slight pause (15s timeout) for watch to trigger
    for (int i = 0; i < (5 * 15); i++) {
        if (zkController2.getClusterState().getLiveNodes().size() == 2) {
            break;
        }
        Thread.sleep(200);
    }
    assertEquals(2, zkController2.getClusterState().getLiveNodes().size());
    // quickly kill / start client
    container2.getZkController().getZkClient().getSolrZooKeeper().getConnection().disconnect();
    container2.shutdown();
    System.setProperty("hostPort", "1662");
    System.setProperty("solr.data.dir", ClusterStateUpdateTest.this.dataDir2.getAbsolutePath());
    container2 = new CoreContainer(solrHomeDirectory.getAbsolutePath());
    container2.load();
    System.clearProperty("hostPort");
    // pause for watch to trigger
    for (int i = 0; i < 200; i++) {
        if (container1.getZkController().getClusterState().liveNodesContain(container2.getZkController().getNodeName())) {
            break;
        }
        Thread.sleep(100);
    }
    assertTrue(container1.getZkController().getClusterState().liveNodesContain(container2.getZkController().getNodeName()));
// core.close();  // don't close - this core is managed by container1 now
}

Example 57 with ZkNodeProps

use of org.apache.solr.common.cloud.ZkNodeProps in project lucene-solr by apache.

the class TestCollectionAPIs method compareOutput.

static ZkNodeProps compareOutput(final ApiBag apiBag, final String path, final SolrRequest.METHOD method, final String payload, final CoreContainer cc, String expectedOutputMapJson) throws Exception {
    Pair<SolrQueryRequest, SolrQueryResponse> ctx = makeCall(apiBag, path, method, payload, cc);
    ZkNodeProps output = (ZkNodeProps) ctx.second().getValues().get(ZkNodeProps.class.getName());
    Map expected = (Map) fromJSONString(expectedOutputMapJson);
    assertMapEqual(expected, output);
    return output;
}

Example 58 with ZkNodeProps

use of org.apache.solr.common.cloud.ZkNodeProps in project lucene-solr by apache.

the class SolrCmdDistributorTest method testRetryNodeWontRetrySocketError.

private void testRetryNodeWontRetrySocketError() throws Exception {
    final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
    long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
    streamingClients.setExp(Exp.SOCKET_EXCEPTION);
    ArrayList<Node> nodes = new ArrayList<>();
    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    final AtomicInteger retries = new AtomicInteger();
    nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {

        @Override
        public boolean checkRetry() {
            retries.incrementAndGet();
            return true;
        }
    };
    nodes.add(retryNode);
    AddUpdateCommand cmd = new AddUpdateCommand(null);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    ModifiableSolrParams params = new ModifiableSolrParams();
    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
    cmdDistrib.distribAdd(cmd, nodes, params);
    streamingClients.setExp(null);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    // it will checkRetry, but not actually do it...
    assertEquals(1, retries.get());
    long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    // we will get java.net.SocketException: Network is unreachable, which we don't retry on
    assertEquals(numFoundBefore, numFoundAfter);
    assertEquals(1, cmdDistrib.getErrors().size());
}

Example 59 with ZkNodeProps

use of org.apache.solr.common.cloud.ZkNodeProps in project lucene-solr by apache.

the class SolrCmdDistributorTest method test.

@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
    del("*:*");
    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    ModifiableSolrParams params = new ModifiableSolrParams();
    List<Node> nodes = new ArrayList<>();
    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, ((HttpSolrClient) controlClient).getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
    // add one doc to controlClient
    AddUpdateCommand cmd = new AddUpdateCommand(null);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    params = new ModifiableSolrParams();
    cmdDistrib.distribAdd(cmd, nodes, params);
    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    List<Error> errors = cmdDistrib.getErrors();
    assertEquals(errors.toString(), 0, errors.size());
    long numFound = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(1, numFound);
    HttpSolrClient client = (HttpSolrClient) clients.get(0);
    nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, client.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
    // add another 2 docs to control and 3 to client
    cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribAdd(cmd, nodes, params);
    int id2 = id.incrementAndGet();
    AddUpdateCommand cmd2 = new AddUpdateCommand(null);
    cmd2.solrDoc = sdoc("id", id2);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribAdd(cmd2, nodes, params);
    AddUpdateCommand cmd3 = new AddUpdateCommand(null);
    cmd3.solrDoc = sdoc("id", id.incrementAndGet());
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribAdd(cmd3, Collections.singletonList(nodes.get(1)), params);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    errors = cmdDistrib.getErrors();
    assertEquals(errors.toString(), 0, errors.size());
    SolrDocumentList results = controlClient.query(new SolrQuery("*:*")).getResults();
    numFound = results.getNumFound();
    assertEquals(results.toString(), 3, numFound);
    numFound = client.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(3, numFound);
    // now delete doc 2 which is on both control and client1
    DeleteUpdateCommand dcmd = new DeleteUpdateCommand(null);
    dcmd.id = Integer.toString(id2);
    cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribDelete(dcmd, nodes, params);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    errors = cmdDistrib.getErrors();
    assertEquals(errors.toString(), 0, errors.size());
    results = controlClient.query(new SolrQuery("*:*")).getResults();
    numFound = results.getNumFound();
    assertEquals(results.toString(), 2, numFound);
    numFound = client.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(results.toString(), 2, numFound);
    for (SolrClient c : clients) {
        c.optimize();
    //System.out.println(clients.get(0).request(new LukeRequest()));
    }
    cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    int cnt = atLeast(303);
    for (int i = 0; i < cnt; i++) {
        nodes.clear();
        for (SolrClient c : clients) {
            if (random().nextBoolean()) {
                continue;
            }
            HttpSolrClient httpClient = (HttpSolrClient) c;
            nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, httpClient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
            nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
        }
        AddUpdateCommand c = new AddUpdateCommand(null);
        c.solrDoc = sdoc("id", id.incrementAndGet());
        if (nodes.size() > 0) {
            params = new ModifiableSolrParams();
            cmdDistrib.distribAdd(c, nodes, params);
        }
    }
    nodes.clear();
    for (SolrClient c : clients) {
        HttpSolrClient httpClient = (HttpSolrClient) c;
        nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, httpClient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
        nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
    }
    final AtomicInteger commits = new AtomicInteger();
    for (JettySolrRunner jetty : jettys) {
        CoreContainer cores = jetty.getCoreContainer();
        try (SolrCore core = cores.getCore("collection1")) {
            core.getUpdateHandler().registerCommitCallback(new SolrEventListener() {

                @Override
                public void init(NamedList args) {
                }

                @Override
                public void postSoftCommit() {
                }

                @Override
                public void postCommit() {
                    commits.incrementAndGet();
                }

                @Override
                public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) {
                }
            });
        }
    }
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    assertEquals(getShardCount(), commits.get());
    for (SolrClient c : clients) {
        NamedList<Object> resp = c.request(new LukeRequest());
        assertEquals("SOLR-3428: We only did adds - there should be no deletes", ((NamedList<Object>) resp.get("index")).get("numDocs"), ((NamedList<Object>) resp.get("index")).get("maxDoc"));
    }
    testMaxRetries();
    testOneRetry();
    testRetryNodeAgainstBadAddress();
    testRetryNodeWontRetrySocketError();
    testDistribOpenSearcher();
}

Example 60 with ZkNodeProps

use of org.apache.solr.common.cloud.ZkNodeProps in project lucene-solr by apache.

the class SolrCmdDistributorTest method testRetryNodeAgainstBadAddress.

private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
    // Test RetryNode
    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
    long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    ArrayList<Node> nodes = new ArrayList<>();
    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
    RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {

        @Override
        public boolean checkRetry() {
            ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
            this.nodeProps = new ZkCoreNodeProps(leaderProps);
            return true;
        }
    };
    nodes.add(retryNode);
    AddUpdateCommand cmd = new AddUpdateCommand(null);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    ModifiableSolrParams params = new ModifiableSolrParams();
    cmdDistrib.distribAdd(cmd, nodes, params);
    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    // different OS's will throw different exceptions for the bad address above
    if (numFoundBefore != numFoundAfter) {
        assertEquals(0, cmdDistrib.getErrors().size());
        assertEquals(numFoundBefore + 1, numFoundAfter);
    } else {
        // we will get java.net.SocketException: Network is unreachable and not retry
        assertEquals(numFoundBefore, numFoundAfter);
        assertEquals(1, cmdDistrib.getErrors().size());
    }
}