Examples with RequestStatusState org.apache.solr.client.solrj.response.RequestStatusState used on opensource projects

Example 6 with RequestStatusState

use of org.apache.solr.client.solrj.response.RequestStatusState in project lucene-solr by apache.

the class CloudSolrClientTest method testNonRetryableRequests.

@Test
public void testNonRetryableRequests() throws Exception {
    try (CloudSolrClient client = getCloudSolrClient(cluster.getZkServer().getZkAddress())) {
        // important to have one replica on each node
        RequestStatusState state = CollectionAdminRequest.createCollection("foo", "conf", 1, NODE_COUNT).processAndWait(client, 60);
        if (state == RequestStatusState.COMPLETED) {
            AbstractDistribZkTestBase.waitForRecoveriesToFinish("foo", client.getZkStateReader(), true, true, TIMEOUT);
            client.setDefaultCollection("foo");
            Map<String, String> adminPathToMbean = new HashMap<>(CommonParams.ADMIN_PATHS.size());
            adminPathToMbean.put(CommonParams.COLLECTIONS_HANDLER_PATH, CollectionsHandler.class.getName());
            adminPathToMbean.put(CommonParams.CORES_HANDLER_PATH, CoreAdminHandler.class.getName());
            adminPathToMbean.put(CommonParams.CONFIGSETS_HANDLER_PATH, ConfigSetsHandler.class.getName());
            for (String adminPath : adminPathToMbean.keySet()) {
                long errorsBefore = 0;
                for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
                    Long numRequests = getNumRequests(runner.getBaseUrl().toString(), "foo", "ADMIN", adminPathToMbean.get(adminPath), adminPath, true);
                    errorsBefore += numRequests;
                    log.info("Found {} requests to {} on {}", numRequests, adminPath, runner.getBaseUrl());
                }
                ModifiableSolrParams params = new ModifiableSolrParams();
                params.set("qt", adminPath);
                // this should cause an error
                params.set("action", "foobar");
                QueryRequest req = new QueryRequest(params);
                try {
                    NamedList<Object> resp = client.request(req);
                    fail("call to foo for admin path " + adminPath + " should have failed");
                } catch (Exception e) {
                // expected
                }
                long errorsAfter = 0;
                for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
                    Long numRequests = getNumRequests(runner.getBaseUrl().toString(), "foo", "ADMIN", adminPathToMbean.get(adminPath), adminPath, true);
                    errorsAfter += numRequests;
                    log.info("Found {} requests to {} on {}", numRequests, adminPath, runner.getBaseUrl());
                }
                assertEquals(errorsBefore + 1, errorsAfter);
            }
        } else {
            fail("Collection could not be created within 60 seconds");
        }
    }
}

Example 7 with RequestStatusState

use of org.apache.solr.client.solrj.response.RequestStatusState in project lucene-solr by apache.

the class ShardSplitTest method testSplitWithChaosMonkey.

@Test
public void testSplitWithChaosMonkey() throws Exception {
    waitForThingsToLevelOut(15);
    List<StoppableIndexingThread> indexers = new ArrayList<>();
    try {
        for (int i = 0; i < 1; i++) {
            StoppableIndexingThread thread = new StoppableIndexingThread(controlClient, cloudClient, String.valueOf(i), true);
            indexers.add(thread);
            thread.start();
        }
        // give the indexers some time to do their work
        Thread.sleep(1000);
    } catch (Exception e) {
        log.error("Error in test", e);
    } finally {
        for (StoppableIndexingThread indexer : indexers) {
            indexer.safeStop();
            indexer.join();
        }
    }
    cloudClient.commit();
    controlClient.commit();
    AtomicBoolean stop = new AtomicBoolean();
    AtomicBoolean killed = new AtomicBoolean(false);
    Runnable monkey = new Runnable() {

        @Override
        public void run() {
            ZkStateReader zkStateReader = cloudClient.getZkStateReader();
            zkStateReader.registerCollectionStateWatcher(AbstractDistribZkTestBase.DEFAULT_COLLECTION, new CollectionStateWatcher() {

                @Override
                public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
                    if (stop.get()) {
                        // abort and remove the watch
                        return true;
                    }
                    Slice slice = collectionState.getSlice(SHARD1_0);
                    if (slice != null && slice.getReplicas().size() > 1) {
                        // ensure that only one watcher invocation thread can kill!
                        if (killed.compareAndSet(false, true)) {
                            log.info("Monkey thread found 2 replicas for {} {}", AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
                            CloudJettyRunner cjetty = shardToLeaderJetty.get(SHARD1);
                            try {
                                Thread.sleep(1000 + random().nextInt(500));
                                ChaosMonkey.kill(cjetty);
                                stop.set(true);
                                return true;
                            } catch (Exception e) {
                                log.error("Monkey unable to kill jetty at port " + cjetty.jetty.getLocalPort(), e);
                            }
                        }
                    }
                    log.info("Monkey thread found only one replica for {} {}", AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1);
                    return false;
                }
            });
        }
    };
    Thread monkeyThread = null;
    /*
     somehow the cluster state object inside this zk state reader has static copy of the collection which is never updated
     so any call to waitForRecoveriesToFinish just keeps looping until timeout.
     We workaround by explicitly registering the collection as an interesting one so that it is watched by ZkStateReader
     see SOLR-9440. Todo remove this hack after SOLR-9440 is fixed.
    */
    cloudClient.getZkStateReader().registerCore(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
    monkeyThread = new Thread(monkey);
    monkeyThread.start();
    try {
        CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
        splitShard.setShardName(SHARD1);
        String asyncId = splitShard.processAsync(cloudClient);
        RequestStatusState splitStatus = null;
        try {
            splitStatus = CollectionAdminRequest.requestStatus(asyncId).waitFor(cloudClient, 120);
        } catch (Exception e) {
            log.warn("Failed to get request status, maybe because the overseer node was shutdown by monkey", e);
        }
        // we don't care if the split failed because we are injecting faults and it is likely
        // that the split has failed but in any case we want to assert that all docs that got
        // indexed are available in SolrCloud and if the split succeeded then all replicas of the sub-shard
        // must be consistent (i.e. have same numdocs)
        log.info("Shard split request state is COMPLETED");
        stop.set(true);
        monkeyThread.join();
        Set<String> addFails = new HashSet<>();
        Set<String> deleteFails = new HashSet<>();
        for (StoppableIndexingThread indexer : indexers) {
            addFails.addAll(indexer.getAddFails());
            deleteFails.addAll(indexer.getDeleteFails());
        }
        CloudJettyRunner cjetty = shardToLeaderJetty.get(SHARD1);
        log.info("Starting shard1 leader jetty at port {}", cjetty.jetty.getLocalPort());
        ChaosMonkey.start(cjetty.jetty);
        cloudClient.getZkStateReader().forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
        log.info("Current collection state: {}", printClusterStateInfo(AbstractDistribZkTestBase.DEFAULT_COLLECTION));
        boolean replicaCreationsFailed = false;
        if (splitStatus == RequestStatusState.FAILED) {
            // either one or more replica creation failed (because it may have been created on the same parent shard leader node)
            // or the split may have failed while trying to soft-commit *after* all replicas have been created
            // the latter counts as a successful switch even if the API doesn't say so
            // so we must find a way to distinguish between the two
            // an easy way to do that is to look at the sub-shard replicas and check if the replica core actually exists
            // instead of existing solely inside the cluster state
            DocCollection collectionState = cloudClient.getZkStateReader().getClusterState().getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
            Slice slice10 = collectionState.getSlice(SHARD1_0);
            Slice slice11 = collectionState.getSlice(SHARD1_1);
            if (slice10 != null && slice11 != null) {
                for (Replica replica : slice10) {
                    if (!doesReplicaCoreExist(replica)) {
                        replicaCreationsFailed = true;
                        break;
                    }
                }
                for (Replica replica : slice11) {
                    if (!doesReplicaCoreExist(replica)) {
                        replicaCreationsFailed = true;
                        break;
                    }
                }
            }
        }
        // true if sub-shard states switch to 'active' eventually
        AtomicBoolean areSubShardsActive = new AtomicBoolean(false);
        if (!replicaCreationsFailed) {
            // all sub-shard replicas were created successfully so all cores must recover eventually
            waitForRecoveriesToFinish(AbstractDistribZkTestBase.DEFAULT_COLLECTION, true);
            // let's wait for the overseer to switch shard states
            CountDownLatch latch = new CountDownLatch(1);
            cloudClient.getZkStateReader().registerCollectionStateWatcher(AbstractDistribZkTestBase.DEFAULT_COLLECTION, new CollectionStateWatcher() {

                @Override
                public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
                    Slice parent = collectionState.getSlice(SHARD1);
                    Slice slice10 = collectionState.getSlice(SHARD1_0);
                    Slice slice11 = collectionState.getSlice(SHARD1_1);
                    if (slice10 != null && slice11 != null && parent.getState() == Slice.State.INACTIVE && slice10.getState() == Slice.State.ACTIVE && slice11.getState() == Slice.State.ACTIVE) {
                        areSubShardsActive.set(true);
                        latch.countDown();
                        // removes the watch
                        return true;
                    } else if (slice10 != null && slice11 != null && parent.getState() == Slice.State.ACTIVE && slice10.getState() == Slice.State.RECOVERY_FAILED && slice11.getState() == Slice.State.RECOVERY_FAILED) {
                        areSubShardsActive.set(false);
                        latch.countDown();
                        return true;
                    }
                    return false;
                }
            });
            latch.await(2, TimeUnit.MINUTES);
            if (latch.getCount() != 0) {
                // sanity check
                fail("We think that split was successful but sub-shard states were not updated even after 2 minutes.");
            }
        }
        // for visibility of results on sub-shards
        cloudClient.commit();
        checkShardConsistency(true, true, addFails, deleteFails);
        long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
        // ensure we have added more than 0 docs
        long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
        assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);
        assertEquals("Found " + ctrlDocs + " control docs and " + cloudClientDocs + " cloud docs", ctrlDocs, cloudClientDocs);
        // handle new shards/replica so well.
        if (areSubShardsActive.get()) {
            ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
            DocCollection collection = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
            int numReplicasChecked = assertConsistentReplicas(collection.getSlice(SHARD1_0));
            assertEquals("We should have checked consistency for exactly 2 replicas of shard1_0", 2, numReplicasChecked);
            numReplicasChecked = assertConsistentReplicas(collection.getSlice(SHARD1_1));
            assertEquals("We should have checked consistency for exactly 2 replicas of shard1_1", 2, numReplicasChecked);
        }
    } finally {
        stop.set(true);
        monkeyThread.join();
    }
}

Example 8 with RequestStatusState

use of org.apache.solr.client.solrj.response.RequestStatusState in project lucene-solr by apache.

the class AsyncCallRequestStatusResponseTest method testAsyncCallStatusResponse.

@Test
public void testAsyncCallStatusResponse() throws Exception {
    String asyncId = CollectionAdminRequest.createCollection("asynccall", "conf", 2, 1).processAsync(cluster.getSolrClient());
    waitForState("Expected collection 'asynccall' to have 2 shards and 1 replica", "asynccall", clusterShape(2, 1));
    int tries = 0;
    while (true) {
        final RequestStatusState state = CollectionAdminRequest.requestStatus(asyncId).process(cluster.getSolrClient()).getRequestStatus();
        if (state == RequestStatusState.COMPLETED)
            break;
        if (tries++ > 10)
            fail("Expected to see RequestStatusState.COMPLETED but was " + state.toString());
        TimeUnit.SECONDS.sleep(1);
    }
    CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId);
    CollectionAdminResponse rsp = requestStatus.process(cluster.getSolrClient());
    NamedList<?> r = rsp.getResponse();
    // Check that there's more response than the hardcoded status and states
    assertEquals("Assertion Failure" + r.toString(), 5, r.size());
}

Example 9 with RequestStatusState

use of org.apache.solr.client.solrj.response.RequestStatusState in project lucene-solr by apache.

the class MultiThreadedOCPTest method testTaskExclusivity.

private void testTaskExclusivity() throws Exception, SolrServerException {
    DistributedQueue distributedQueue = new DistributedQueue(cloudClient.getZkStateReader().getZkClient(), "/overseer/collection-queue-work", new Overseer.Stats());
    try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
        Create createCollectionRequest = CollectionAdminRequest.createCollection("ocptest_shardsplit", "conf1", 4, 1);
        createCollectionRequest.processAsync("1000", client);
        distributedQueue.offer(Utils.toJSON(Utils.makeMap("collection", "ocptest_shardsplit", QUEUE_OPERATION, MOCK_COLL_TASK.toLower(), ASYNC, "1001", "sleep", "100")));
        distributedQueue.offer(Utils.toJSON(Utils.makeMap("collection", "ocptest_shardsplit", QUEUE_OPERATION, MOCK_COLL_TASK.toLower(), ASYNC, "1002", "sleep", "100")));
        int iterations = 0;
        while (true) {
            int runningTasks = 0;
            int completedTasks = 0;
            for (int i = 1001; i <= 1002; i++) {
                final RequestStatusState state = getRequestState(i, client);
                if (state == RequestStatusState.RUNNING) {
                    runningTasks++;
                } else if (state == RequestStatusState.COMPLETED) {
                    completedTasks++;
                }
                assertNotSame("We have a failed SPLITSHARD task", RequestStatusState.FAILED, state);
            }
            // TODO: REQUESTSTATUS might come back with more than 1 running tasks over multiple calls.
            // The only way to fix this is to support checking of multiple requestids in a single REQUESTSTATUS task.
            assertTrue("Mutual exclusion failed. Found more than one task running for the same collection", runningTasks < 2);
            if (completedTasks == 2 || iterations++ > REQUEST_STATUS_TIMEOUT)
                break;
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                return;
            }
        }
        for (int i = 1001; i <= 1002; i++) {
            final RequestStatusState state = getRequestStateAfterCompletion(i + "", REQUEST_STATUS_TIMEOUT, client);
            assertSame("Task " + i + " did not complete, final state: " + state, RequestStatusState.COMPLETED, state);
        }
    }
}

Example 10 with RequestStatusState

use of org.apache.solr.client.solrj.response.RequestStatusState in project lucene-solr by apache.

the class MultiThreadedOCPTest method testParallelCollectionAPICalls.

private void testParallelCollectionAPICalls() throws IOException, SolrServerException {
    try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
        for (int i = 1; i <= NUM_COLLECTIONS; i++) {
            CollectionAdminRequest.createCollection("ocptest" + i, "conf1", 4, 1).processAsync(String.valueOf(i), client);
        }
        boolean pass = false;
        int counter = 0;
        while (true) {
            int numRunningTasks = 0;
            for (int i = 1; i <= NUM_COLLECTIONS; i++) if (getRequestState(i + "", client) == RequestStatusState.RUNNING) {
                numRunningTasks++;
            }
            if (numRunningTasks > 1) {
                pass = true;
                break;
            } else if (counter++ > 100) {
                break;
            }
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
        assertTrue("More than one tasks were supposed to be running in parallel but they weren't.", pass);
        for (int i = 1; i <= NUM_COLLECTIONS; i++) {
            final RequestStatusState state = getRequestStateAfterCompletion(i + "", REQUEST_STATUS_TIMEOUT, client);
            assertSame("Task " + i + " did not complete, final state: " + state, RequestStatusState.COMPLETED, state);
        }
    }
}