Examples with SolrClient org.apache.solr.client.solrj.SolrClient used on opensource projects

Example 21 with SolrClient

use of org.apache.solr.client.solrj.SolrClient in project lucene-solr by apache.

the class TestRandomFlRTGCloud method assertRTG.

/**
   * Does one or more RTG request for the specified docIds with a randomized fl & fq params, asserting
   * that the returned document (if any) makes sense given the expected SolrInputDocuments
   */
private void assertRTG(final SolrInputDocument[] knownDocs, final int[] docIds) throws IOException, SolrServerException {
    final SolrClient client = getRandClient(random());
    // NOTE: not using SolrClient.getById or getByIds because we want to force choice of "id" vs "ids" params
    final ModifiableSolrParams params = params("qt", "/get");
    // random fq -- nothing fancy, secondary concern for our test
    final Integer FQ_MAX = usually() ? null : random().nextInt();
    if (null != FQ_MAX) {
        params.add("fq", "aaa_i:[* TO " + FQ_MAX + "]");
    }
    final Set<FlValidator> validators = new LinkedHashSet<>();
    // always include id so we can be confident which doc we're looking at
    validators.add(ID_VALIDATOR);
    addRandomFlValidators(random(), validators);
    FlValidator.addParams(validators, params);
    final List<String> idsToRequest = new ArrayList<>(docIds.length);
    final List<SolrInputDocument> docsToExpect = new ArrayList<>(docIds.length);
    for (int docId : docIds) {
        // every docId will be included in the request
        idsToRequest.add("" + docId);
        // only docs that should actually exist and match our (optional) filter will be expected in response
        if (null != knownDocs[docId]) {
            Integer filterVal = (Integer) knownDocs[docId].getFieldValue("aaa_i");
            if (null == FQ_MAX || ((null != filterVal) && filterVal.intValue() <= FQ_MAX.intValue())) {
                docsToExpect.add(knownDocs[docId]);
            }
        }
    }
    // even w/only 1 docId requested, the response format can vary depending on how we request it
    final boolean askForList = random().nextBoolean() || (1 != idsToRequest.size());
    if (askForList) {
        if (1 == idsToRequest.size()) {
            // have to be careful not to try to use "multi" 'id' params with only 1 docId
            // with a single docId, the only way to ask for a list is with the "ids" param
            params.add("ids", idsToRequest.get(0));
        } else {
            if (random().nextBoolean()) {
                // each id in it's own param
                for (String id : idsToRequest) {
                    params.add("id", id);
                }
            } else {
                // add one or more comma separated ids params
                params.add(buildCommaSepParams(random(), "ids", idsToRequest));
            }
        }
    } else {
        assert 1 == idsToRequest.size();
        params.add("id", idsToRequest.get(0));
    }
    final QueryResponse rsp = client.query(params);
    assertNotNull(params.toString(), rsp);
    final SolrDocumentList docs = getDocsFromRTGResponse(askForList, rsp);
    assertNotNull(params + " => " + rsp, docs);
    assertEquals("num docs mismatch: " + params + " => " + docsToExpect + " vs " + docs, docsToExpect.size(), docs.size());
    // NOTE: RTG makes no garuntees about the order docs will be returned in when multi requested
    for (SolrDocument actual : docs) {
        try {
            int actualId = assertParseInt("id", actual.getFirstValue("id"));
            final SolrInputDocument expected = knownDocs[actualId];
            assertNotNull("expected null doc but RTG returned: " + actual, expected);
            Set<String> expectedFieldNames = new TreeSet<>();
            for (FlValidator v : validators) {
                expectedFieldNames.addAll(v.assertRTGResults(validators, expected, actual));
            }
            // ensure only expected field names are in the actual document
            Set<String> actualFieldNames = new TreeSet<>(actual.getFieldNames());
            assertEquals("Actual field names returned differs from expected", expectedFieldNames, actualFieldNames);
        } catch (AssertionError ae) {
            throw new AssertionError(params + " => " + actual + ": " + ae.getMessage(), ae);
        }
    }
}

Example 22 with SolrClient

use of org.apache.solr.client.solrj.SolrClient in project lucene-solr by apache.

the class TestTlogReplica method checkRTG.

private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception {
    for (JettySolrRunner solrRunner : solrRunners) {
        try (SolrClient client = solrRunner.newClient()) {
            for (int i = from; i <= to; i++) {
                SolrQuery query = new SolrQuery();
                query.set("distrib", false);
                query.setRequestHandler("/get");
                query.set("id", i);
                QueryResponse res = client.query(collectionName, query);
                assertNotNull("Can not find doc " + i + " in " + solrRunner.getBaseUrl(), res.getResponse().get("doc"));
            }
        }
    }
}

Example 23 with SolrClient

use of org.apache.solr.client.solrj.SolrClient in project lucene-solr by apache.

the class TestTlogReplica method testOutOfOrderDBQWithInPlaceUpdates.

public void testOutOfOrderDBQWithInPlaceUpdates() throws Exception {
    createAndWaitForCollection(1, 0, 2, 0);
    assertFalse(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").indexed());
    assertFalse(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").stored());
    assertTrue(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").hasDocValues());
    List<UpdateRequest> updates = new ArrayList<>();
    // full update
    updates.add(simulatedUpdateRequest(null, "id", 1, "title_s", "title0_new", "inplace_updatable_int", 5, "_version_", 1L));
    updates.add(simulatedDBQ("inplace_updatable_int:5", 3L));
    updates.add(simulatedUpdateRequest(1L, "id", 1, "inplace_updatable_int", 6, "_version_", 2L));
    for (JettySolrRunner solrRunner : getSolrRunner(false)) {
        try (SolrClient client = solrRunner.newClient()) {
            for (UpdateRequest up : updates) {
                up.process(client, collectionName);
            }
        }
    }
    JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
    ChaosMonkey.kill(oldLeaderJetty);
    waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));
    ChaosMonkey.start(oldLeaderJetty);
    waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
    checkRTG(1, 1, cluster.getJettySolrRunners());
    SolrDocument doc = cluster.getSolrClient().getById(collectionName, "1");
    assertNotNull(doc.get("title_s"));
}

Example 24 with SolrClient

use of org.apache.solr.client.solrj.SolrClient in project lucene-solr by apache.

the class TestSolrCloudSnapshots method testSnapshots.

@Test
public void testSnapshots() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "SolrCloudSnapshots";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", NUM_SHARDS, NUM_REPLICAS);
    create.process(solrClient);
    int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed);
    BackupRestoreUtils.verifyDocs(nDocs, solrClient, collectionName);
    String commitName = TestUtil.randomSimpleString(random(), 1, 5);
    // Verify if snapshot creation works with replica failures.
    boolean replicaFailures = usually();
    Optional<String> stoppedCoreName = Optional.empty();
    if (replicaFailures) {
        // Here the assumption is that Solr will spread the replicas uniformly across nodes.
        // If this is not true for some reason, then we will need to add some logic to find a
        // node with a single replica.
        this.cluster.getRandomJetty(random()).stop();
        // Sleep a bit for allowing ZK watch to fire.
        Thread.sleep(5000);
        // Figure out if at-least one replica is "down".
        DocCollection collState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
        for (Slice s : collState.getSlices()) {
            for (Replica replica : s.getReplicas()) {
                if (replica.getState() == State.DOWN) {
                    stoppedCoreName = Optional.of(replica.getCoreName());
                }
            }
        }
    }
    int expectedCoresWithSnapshot = stoppedCoreName.isPresent() ? (NUM_SHARDS * NUM_REPLICAS) - 1 : (NUM_SHARDS * NUM_REPLICAS);
    CollectionAdminRequest.CreateSnapshot createSnap = new CollectionAdminRequest.CreateSnapshot(collectionName, commitName);
    createSnap.process(solrClient);
    Collection<CollectionSnapshotMetaData> collectionSnaps = listCollectionSnapshots(solrClient, collectionName);
    assertEquals(1, collectionSnaps.size());
    CollectionSnapshotMetaData meta = collectionSnaps.iterator().next();
    assertEquals(commitName, meta.getName());
    assertEquals(CollectionSnapshotMetaData.SnapshotStatus.Successful, meta.getStatus());
    assertEquals(expectedCoresWithSnapshot, meta.getReplicaSnapshots().size());
    Map<String, CoreSnapshotMetaData> snapshotByCoreName = meta.getReplicaSnapshots().stream().collect(Collectors.toMap(CoreSnapshotMetaData::getCoreName, Function.identity()));
    DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
    assertEquals(2, collectionState.getActiveSlices().size());
    for (Slice shard : collectionState.getActiveSlices()) {
        assertEquals(2, shard.getReplicas().size());
        for (Replica replica : shard.getReplicas()) {
            if (stoppedCoreName.isPresent() && stoppedCoreName.get().equals(replica.getCoreName())) {
                // We know that the snapshot is not created for this replica.
                continue;
            }
            String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
            String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
            assertTrue(snapshotByCoreName.containsKey(coreName));
            CoreSnapshotMetaData coreSnapshot = snapshotByCoreName.get(coreName);
            try (SolrClient adminClient = getHttpSolrClient(replicaBaseUrl)) {
                Collection<SnapshotMetaData> snapshots = listCoreSnapshots(adminClient, coreName);
                Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
                assertTrue("Snapshot not created for core " + coreName, metaData.isPresent());
                assertEquals(coreSnapshot.getIndexDirPath(), metaData.get().getIndexDirPath());
                assertEquals(coreSnapshot.getGenerationNumber(), metaData.get().getGenerationNumber());
            }
        }
    }
    // Delete all documents.
    {
        solrClient.deleteByQuery(collectionName, "*:*");
        solrClient.commit(collectionName);
        BackupRestoreUtils.verifyDocs(0, solrClient, collectionName);
    }
    String backupLocation = createTempDir().toFile().getAbsolutePath();
    String backupName = "mytestbackup";
    String restoreCollectionName = collectionName + "_restored";
    //Create a backup using the earlier created snapshot.
    {
        CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName).setLocation(backupLocation).setCommitName(commitName);
        if (random().nextBoolean()) {
            assertEquals(0, backup.process(solrClient).getStatus());
        } else {
            //async
            assertEquals(RequestStatusState.COMPLETED, backup.processAndWait(solrClient, 30));
        }
    }
    // Restore backup.
    {
        CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName).setLocation(backupLocation);
        if (replicaFailures) {
            // In this case one of the Solr servers would be down. Hence we need to increase
            // max_shards_per_node property for restore command to succeed.
            restore.setMaxShardsPerNode(2);
        }
        if (random().nextBoolean()) {
            assertEquals(0, restore.process(solrClient).getStatus());
        } else {
            //async
            assertEquals(RequestStatusState.COMPLETED, restore.processAndWait(solrClient, 30));
        }
        AbstractDistribZkTestBase.waitForRecoveriesToFinish(restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
        BackupRestoreUtils.verifyDocs(nDocs, solrClient, restoreCollectionName);
    }
    // Verify if the snapshot deletion works correctly when one or more replicas containing the snapshot are
    // deleted
    boolean replicaDeletion = rarely();
    if (replicaDeletion) {
        CoreSnapshotMetaData replicaToDelete = null;
        for (String shardId : meta.getShards()) {
            List<CoreSnapshotMetaData> replicas = meta.getReplicaSnapshotsForShard(shardId);
            if (replicas.size() > 1) {
                int r_index = random().nextInt(replicas.size());
                replicaToDelete = replicas.get(r_index);
            }
        }
        if (replicaToDelete != null) {
            collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
            for (Slice s : collectionState.getSlices()) {
                for (Replica r : s.getReplicas()) {
                    if (r.getCoreName().equals(replicaToDelete.getCoreName())) {
                        log.info("Deleting replica {}", r);
                        CollectionAdminRequest.DeleteReplica delReplica = CollectionAdminRequest.deleteReplica(collectionName, replicaToDelete.getShardId(), r.getName());
                        delReplica.process(solrClient);
                        // The replica deletion will cleanup the snapshot meta-data.
                        snapshotByCoreName.remove(r.getCoreName());
                        break;
                    }
                }
            }
        }
    }
    // Delete snapshot
    CollectionAdminRequest.DeleteSnapshot deleteSnap = new CollectionAdminRequest.DeleteSnapshot(collectionName, commitName);
    deleteSnap.process(solrClient);
    // Wait for a while so that the clusterstate.json updates are propagated to the client side.
    Thread.sleep(2000);
    collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
    for (Slice shard : collectionState.getActiveSlices()) {
        for (Replica replica : shard.getReplicas()) {
            if (stoppedCoreName.isPresent() && stoppedCoreName.get().equals(replica.getCoreName())) {
                // We know that the snapshot was not created for this replica.
                continue;
            }
            String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
            String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
            try (SolrClient adminClient = getHttpSolrClient(replicaBaseUrl)) {
                Collection<SnapshotMetaData> snapshots = listCoreSnapshots(adminClient, coreName);
                Optional<SnapshotMetaData> metaData = snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst();
                assertFalse("Snapshot not deleted for core " + coreName, metaData.isPresent());
                // Remove the entry for core if the snapshot is deleted successfully.
                snapshotByCoreName.remove(coreName);
            }
        }
    }
    // Verify all core-level snapshots are deleted.
    assertTrue("The cores remaining " + snapshotByCoreName, snapshotByCoreName.isEmpty());
    assertTrue(listCollectionSnapshots(solrClient, collectionName).isEmpty());
    // Verify if the collection deletion result in proper cleanup of snapshot metadata.
    {
        String commitName_2 = commitName + "_2";
        CollectionAdminRequest.CreateSnapshot createSnap_2 = new CollectionAdminRequest.CreateSnapshot(collectionName, commitName_2);
        assertEquals(0, createSnap_2.process(solrClient).getStatus());
        Collection<CollectionSnapshotMetaData> collectionSnaps_2 = listCollectionSnapshots(solrClient, collectionName);
        assertEquals(1, collectionSnaps.size());
        assertEquals(commitName_2, collectionSnaps_2.iterator().next().getName());
        // Delete collection
        CollectionAdminRequest.Delete deleteCol = CollectionAdminRequest.deleteCollection(collectionName);
        assertEquals(0, deleteCol.process(solrClient).getStatus());
        assertTrue(SolrSnapshotManager.listSnapshots(solrClient.getZkStateReader().getZkClient(), collectionName).isEmpty());
    }
}

Example 25 with SolrClient

use of org.apache.solr.client.solrj.SolrClient in project lucene-solr by apache.

the class TestSolrCoreSnapshots method deleteSnapshot.

private void deleteSnapshot(SolrClient adminClient, String coreName, String commitName) throws Exception {
    DeleteSnapshot req = new DeleteSnapshot(commitName);
    req.setCoreName(coreName);
    adminClient.request(req);
    Collection<SnapshotMetaData> snapshots = listSnapshots(adminClient, coreName);
    assertFalse(snapshots.stream().filter(x -> commitName.equals(x.getName())).findFirst().isPresent());
}