Examples with Unload org.apache.solr.client.solrj.request.CoreAdminRequest.Unload used on opensource projects

Example 1 with Unload

use of org.apache.solr.client.solrj.request.CoreAdminRequest.Unload in project lucene-solr by apache.

the class BasicDistributedZkTest method testANewCollectionInOneInstanceWithManualShardAssignement.

private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
    log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");
    System.clearProperty("numShards");
    List<SolrClient> collectionClients = new ArrayList<>();
    SolrClient client = clients.get(0);
    final String baseUrl = ((HttpSolrClient) client).getBaseURL().substring(0, ((HttpSolrClient) client).getBaseURL().length() - DEFAULT_COLLECTION.length() - 1);
    createSolrCore(oneInstanceCollection2, collectionClients, baseUrl, 1, "slice1");
    createSolrCore(oneInstanceCollection2, collectionClients, baseUrl, 2, "slice2");
    createSolrCore(oneInstanceCollection2, collectionClients, baseUrl, 3, "slice2");
    createSolrCore(oneInstanceCollection2, collectionClients, baseUrl, 4, "slice1");
    while (pending != null && pending.size() > 0) {
        Future<Object> future = completionService.take();
        pending.remove(future);
    }
    SolrClient client1 = collectionClients.get(0);
    SolrClient client2 = collectionClients.get(1);
    SolrClient client3 = collectionClients.get(2);
    SolrClient client4 = collectionClients.get(3);
    // no one should be recovering
    waitForRecoveriesToFinish(oneInstanceCollection2, getCommonCloudSolrClient().getZkStateReader(), false, true);
    assertAllActive(oneInstanceCollection2, getCommonCloudSolrClient().getZkStateReader());
    //printLayout();
    // TODO: enable when we don't falsely get slice1...
    // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000);
    // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000);
    client2.add(getDoc(id, "1"));
    client3.add(getDoc(id, "2"));
    client4.add(getDoc(id, "3"));
    client1.commit();
    SolrQuery query = new SolrQuery("*:*");
    query.set("distrib", false);
    long oneDocs = client1.query(query).getResults().getNumFound();
    long twoDocs = client2.query(query).getResults().getNumFound();
    long threeDocs = client3.query(query).getResults().getNumFound();
    long fourDocs = client4.query(query).getResults().getNumFound();
    query.set("collection", oneInstanceCollection2);
    query.set("distrib", true);
    long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound();
    //    System.out.println("1:" + oneDocs);
    //    System.out.println("2:" + twoDocs);
    //    System.out.println("3:" + threeDocs);
    //    System.out.println("4:" + fourDocs);
    //    System.out.println("All Docs:" + allDocs);
    //    assertEquals(oneDocs, threeDocs);
    //    assertEquals(twoDocs, fourDocs);
    //    assertNotSame(oneDocs, twoDocs);
    assertEquals(3, allDocs);
    // we added a role of none on these creates - check for it
    ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader();
    zkStateReader.forceUpdateCollection(oneInstanceCollection2);
    Map<String, Slice> slices = zkStateReader.getClusterState().getSlicesMap(oneInstanceCollection2);
    assertNotNull(slices);
    String roles = slices.get("slice1").getReplicasMap().values().iterator().next().getStr(ZkStateReader.ROLES_PROP);
    assertEquals("none", roles);
    ZkCoreNodeProps props = new ZkCoreNodeProps(getCommonCloudSolrClient().getZkStateReader().getClusterState().getLeader(oneInstanceCollection2, "slice1"));
    // now test that unloading a core gets us a new leader
    try (HttpSolrClient unloadClient = getHttpSolrClient(baseUrl)) {
        unloadClient.setConnectionTimeout(15000);
        unloadClient.setSoTimeout(60000);
        Unload unloadCmd = new Unload(true);
        unloadCmd.setCoreName(props.getCoreName());
        String leader = props.getCoreUrl();
        unloadClient.request(unloadCmd);
        int tries = 50;
        while (leader.equals(zkStateReader.getLeaderUrl(oneInstanceCollection2, "slice1", 10000))) {
            Thread.sleep(100);
            if (tries-- == 0) {
                fail("Leader never changed");
            }
        }
    }
    IOUtils.close(collectionClients);
}

Example 2 with Unload

use of org.apache.solr.client.solrj.request.CoreAdminRequest.Unload in project lucene-solr by apache.

the class UnloadDistributedZkTest method testUnloadLotsOfCores.

private void testUnloadLotsOfCores() throws Exception {
    SolrClient client = clients.get(2);
    String url3 = getBaseUrl(client);
    try (final HttpSolrClient adminClient = getHttpSolrClient(url3)) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(60000);
        int cnt = atLeast(3);
        ThreadPoolExecutor executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new DefaultSolrThreadFactory("testExecutor"));
        try {
            // create the cores
            createCores(adminClient, executor, "multiunload", 2, cnt);
        } finally {
            ExecutorUtil.shutdownAndAwaitTermination(executor);
        }
        executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new DefaultSolrThreadFactory("testExecutor"));
        try {
            for (int j = 0; j < cnt; j++) {
                final int freezeJ = j;
                executor.execute(() -> {
                    Unload unloadCmd = new Unload(true);
                    unloadCmd.setCoreName("multiunload" + freezeJ);
                    try {
                        adminClient.request(unloadCmd);
                    } catch (SolrServerException | IOException e) {
                        throw new RuntimeException(e);
                    }
                });
                Thread.sleep(random().nextInt(50));
            }
        } finally {
            ExecutorUtil.shutdownAndAwaitTermination(executor);
        }
    }
}

Example 3 with Unload

use of org.apache.solr.client.solrj.request.CoreAdminRequest.Unload in project lucene-solr by apache.

the class UnloadDistributedZkTest method testUnloadShardAndCollection.

private void testUnloadShardAndCollection() throws Exception {
    final int numShards = 2;
    final String collection = "test_unload_shard_and_collection";
    final String coreName1 = collection + "_1";
    final String coreName2 = collection + "_2";
    // create one leader and one replica
    Create createCmd = new Create();
    createCmd.setCoreName(coreName1);
    createCmd.setCollection(collection);
    String coreDataDir = createTempDir().toFile().getAbsolutePath();
    createCmd.setDataDir(getDataDir(coreDataDir));
    createCmd.setNumShards(numShards);
    SolrClient client = clients.get(0);
    String url1 = getBaseUrl(client);
    try (HttpSolrClient adminClient = getHttpSolrClient(url1)) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(60000);
        adminClient.request(createCmd);
        createCmd = new Create();
        createCmd.setCoreName(coreName2);
        createCmd.setCollection(collection);
        coreDataDir = createTempDir().toFile().getAbsolutePath();
        createCmd.setDataDir(getDataDir(coreDataDir));
        adminClient.request(createCmd);
        // does not mean they are active and up yet :*
        waitForRecoveriesToFinish(collection, false);
        final boolean unloadInOrder = random().nextBoolean();
        final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2);
        final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1);
        // now unload one of the two
        Unload unloadCmd = new Unload(false);
        unloadCmd.setCoreName(unloadCmdCoreName1);
        adminClient.request(unloadCmd);
        // there should still be two shards (as of SOLR-5209)
        checkCoreNamePresenceAndSliceCount(collection, unloadCmdCoreName1, false, /* shouldBePresent */
        numShards);
        // now unload one of the other
        unloadCmd = new Unload(false);
        unloadCmd.setCoreName(unloadCmdCoreName2);
        adminClient.request(unloadCmd);
        checkCoreNamePresenceAndSliceCount(collection, unloadCmdCoreName2, false, /* shouldBePresent */
        numShards);
    }
    //printLayout();
    // the collection should still be present (as of SOLR-5209 replica removal does not cascade to remove the slice and collection)
    assertTrue("No longer found collection " + collection, getCommonCloudSolrClient().getZkStateReader().getClusterState().hasCollection(collection));
}

Example 4 with Unload

use of org.apache.solr.client.solrj.request.CoreAdminRequest.Unload in project lucene-solr by apache.

the class UnloadDistributedZkTest method testCoreUnloadAndLeaders.

/**
   * @throws Exception on any problem
   */
private void testCoreUnloadAndLeaders() throws Exception {
    File tmpDir = createTempDir().toFile();
    String core1DataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection1" + "_1n";
    // create a new collection collection
    SolrClient client = clients.get(0);
    String url1 = getBaseUrl(client);
    try (HttpSolrClient adminClient = getHttpSolrClient(url1)) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(60000);
        Create createCmd = new Create();
        createCmd.setCoreName("unloadcollection1");
        createCmd.setCollection("unloadcollection");
        createCmd.setNumShards(1);
        createCmd.setDataDir(getDataDir(core1DataDir));
        adminClient.request(createCmd);
    }
    ZkStateReader zkStateReader = getCommonCloudSolrClient().getZkStateReader();
    zkStateReader.forceUpdateCollection("unloadcollection");
    int slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();
    assertEquals(1, slices);
    client = clients.get(1);
    String url2 = getBaseUrl(client);
    try (HttpSolrClient adminClient = getHttpSolrClient(url2)) {
        Create createCmd = new Create();
        createCmd.setCoreName("unloadcollection2");
        createCmd.setCollection("unloadcollection");
        String core2dataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection1" + "_2n";
        createCmd.setDataDir(getDataDir(core2dataDir));
        adminClient.request(createCmd);
    }
    zkStateReader.forceUpdateCollection("unloadcollection");
    slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size();
    assertEquals(1, slices);
    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
    ZkCoreNodeProps leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1");
    Random random = random();
    if (random.nextBoolean()) {
        try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getCoreUrl())) {
            // lets try and use the solrj client to index and retrieve a couple
            // documents
            SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
            SolrInputDocument doc2 = getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls");
            SolrInputDocument doc3 = getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled");
            collectionClient.add(doc1);
            collectionClient.add(doc2);
            collectionClient.add(doc3);
            collectionClient.commit();
        }
    }
    // create another replica for our collection
    client = clients.get(2);
    String url3 = getBaseUrl(client);
    try (HttpSolrClient adminClient = getHttpSolrClient(url3)) {
        Create createCmd = new Create();
        createCmd.setCoreName("unloadcollection3");
        createCmd.setCollection("unloadcollection");
        String core3dataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection" + "_3n";
        createCmd.setDataDir(getDataDir(core3dataDir));
        adminClient.request(createCmd);
    }
    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
    // so that we start with some versions when we reload...
    DirectUpdateHandler2.commitOnClose = false;
    try (HttpSolrClient addClient = getHttpSolrClient(url3 + "/unloadcollection3")) {
        addClient.setConnectionTimeout(30000);
        // add a few docs
        for (int x = 20; x < 100; x++) {
            SolrInputDocument doc1 = getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
            addClient.add(doc1);
        }
    }
    // unload the leader
    try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getBaseUrl())) {
        collectionClient.setConnectionTimeout(15000);
        collectionClient.setSoTimeout(30000);
        Unload unloadCmd = new Unload(false);
        unloadCmd.setCoreName(leaderProps.getCoreName());
        ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
        collectionClient.request(unloadCmd);
    }
    //    Thread.currentThread().sleep(500);
    //    printLayout();
    int tries = 50;
    while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
        Thread.sleep(100);
        if (tries-- == 0) {
            fail("Leader never changed");
        }
    }
    // ensure there is a leader
    zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
    try (HttpSolrClient addClient = getHttpSolrClient(url2 + "/unloadcollection2")) {
        addClient.setConnectionTimeout(30000);
        addClient.setSoTimeout(90000);
        // add a few docs while the leader is down
        for (int x = 101; x < 200; x++) {
            SolrInputDocument doc1 = getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
            addClient.add(doc1);
        }
    }
    // create another replica for our collection
    client = clients.get(3);
    String url4 = getBaseUrl(client);
    try (HttpSolrClient adminClient = getHttpSolrClient(url4)) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(30000);
        Create createCmd = new Create();
        createCmd.setCoreName("unloadcollection4");
        createCmd.setCollection("unloadcollection");
        String core4dataDir = tmpDir.getAbsolutePath() + File.separator + System.nanoTime() + "unloadcollection" + "_4n";
        createCmd.setDataDir(getDataDir(core4dataDir));
        adminClient.request(createCmd);
    }
    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
    // unload the leader again
    leaderProps = getLeaderUrlFromZk("unloadcollection", "shard1");
    try (HttpSolrClient collectionClient = getHttpSolrClient(leaderProps.getBaseUrl())) {
        collectionClient.setConnectionTimeout(15000);
        collectionClient.setSoTimeout(30000);
        Unload unloadCmd = new Unload(false);
        unloadCmd.setCoreName(leaderProps.getCoreName());
        SolrParams p = (ModifiableSolrParams) unloadCmd.getParams();
        collectionClient.request(unloadCmd);
    }
    tries = 50;
    while (leaderProps.getCoreUrl().equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) {
        Thread.sleep(100);
        if (tries-- == 0) {
            fail("Leader never changed");
        }
    }
    zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
    // set this back
    DirectUpdateHandler2.commitOnClose = true;
    // bring the downed leader back as replica
    try (HttpSolrClient adminClient = getHttpSolrClient(leaderProps.getBaseUrl())) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(30000);
        Create createCmd = new Create();
        createCmd.setCoreName(leaderProps.getCoreName());
        createCmd.setCollection("unloadcollection");
        createCmd.setDataDir(getDataDir(core1DataDir));
        adminClient.request(createCmd);
    }
    waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
    long found1, found3;
    try (HttpSolrClient adminClient = getHttpSolrClient(url2 + "/unloadcollection")) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(30000);
        adminClient.commit();
        SolrQuery q = new SolrQuery("*:*");
        q.set("distrib", false);
        found1 = adminClient.query(q).getResults().getNumFound();
    }
    try (HttpSolrClient adminClient = getHttpSolrClient(url3 + "/unloadcollection")) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(30000);
        adminClient.commit();
        SolrQuery q = new SolrQuery("*:*");
        q.set("distrib", false);
        found3 = adminClient.query(q).getResults().getNumFound();
    }
    try (HttpSolrClient adminClient = getHttpSolrClient(url4 + "/unloadcollection")) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(30000);
        adminClient.commit();
        SolrQuery q = new SolrQuery("*:*");
        q.set("distrib", false);
        long found4 = adminClient.query(q).getResults().getNumFound();
        // all 3 shards should now have the same number of docs
        assertEquals(found1, found3);
        assertEquals(found3, found4);
    }
}