Examples with Replica org.apache.solr.common.cloud.Replica used on opensource projects

Example 56 with Replica

use of org.apache.solr.common.cloud.Replica in project lucene-solr by apache.

the class TestHdfsBackupRestoreCore method test.

@Test
public void test() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "HdfsBackupRestore";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
    create.process(solrClient);
    int nDocs = BackupRestoreUtils.indexDocs(solrClient, collectionName, docsSeed);
    DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
    assertEquals(1, collectionState.getActiveSlices().size());
    Slice shard = collectionState.getActiveSlices().iterator().next();
    assertEquals(1, shard.getReplicas().size());
    Replica replica = shard.getReplicas().iterator().next();
    String replicaBaseUrl = replica.getStr(BASE_URL_PROP);
    String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP);
    String backupName = TestUtil.randomSimpleString(random(), 1, 5);
    boolean testViaReplicationHandler = random().nextBoolean();
    String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();
    try (SolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) {
        // Create a backup.
        if (testViaReplicationHandler) {
            log.info("Running Backup via replication handler");
            BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
            CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, coreName, null);
            while (!checkBackupStatus.success) {
                checkBackupStatus.fetchStatus();
                Thread.sleep(1000);
            }
        } else {
            log.info("Running Backup via core admin api");
            Map<String, String> params = new HashMap<>();
            params.put("name", backupName);
            params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
            BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
        }
        int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
        for (int attempts = 0; attempts < numRestoreTests; attempts++) {
            //Modify existing index before we call restore.
            if (nDocs > 0) {
                //Delete a few docs
                int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
                for (int i = 0; i < numDeletes; i++) {
                    masterClient.deleteByQuery(collectionName, "id:" + i);
                }
                masterClient.commit(collectionName);
                //Add a few more
                int moreAdds = TestUtil.nextInt(random(), 1, 100);
                for (int i = 0; i < moreAdds; i++) {
                    SolrInputDocument doc = new SolrInputDocument();
                    doc.addField("id", i + nDocs);
                    doc.addField("name", "name = " + (i + nDocs));
                    masterClient.add(collectionName, doc);
                }
                //Purposely not calling commit once in a while. There can be some docs which are not committed
                if (usually()) {
                    masterClient.commit(collectionName);
                }
            }
            // Snapshooter prefixes "snapshot." to the backup name.
            if (testViaReplicationHandler) {
                log.info("Running Restore via replication handler");
                // Snapshooter prefixes "snapshot." to the backup name.
                BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName);
                while (!TestRestoreCore.fetchRestoreStatus(baseUrl, coreName)) {
                    Thread.sleep(1000);
                }
            } else {
                log.info("Running Restore via core admin api");
                Map<String, String> params = new HashMap<>();
                params.put("name", "snapshot." + backupName);
                params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
                BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
            }
            //See if restore was successful by checking if all the docs are present again
            BackupRestoreUtils.verifyDocs(nDocs, masterClient, coreName);
            // Verify the permissions for the backup folder.
            FileStatus status = fs.getFileStatus(new org.apache.hadoop.fs.Path("/backup/snapshot." + backupName));
            FsPermission perm = status.getPermission();
            assertEquals(FsAction.ALL, perm.getUserAction());
            assertEquals(FsAction.ALL, perm.getGroupAction());
            assertEquals(FsAction.ALL, perm.getOtherAction());
        }
    }
}

Example 57 with Replica

use of org.apache.solr.common.cloud.Replica in project lucene-solr by apache.

the class TestBlobHandler method doBlobHandlerTest.

@Test
public void doBlobHandlerTest() throws Exception {
    try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
        CollectionAdminResponse response1;
        CollectionAdminRequest.Create createCollectionRequest = CollectionAdminRequest.createCollection(".system", 1, 2);
        response1 = createCollectionRequest.process(client);
        assertEquals(0, response1.getStatus());
        assertTrue(response1.isSuccess());
        DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
        Replica replica = sysColl.getActiveSlicesMap().values().iterator().next().getLeader();
        String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
        String url = baseUrl + "/.system/config/requestHandler";
        Map map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
        assertNotNull(map);
        assertEquals("solr.BlobHandler", getObjectByPath(map, true, Arrays.asList("config", "requestHandler", "/blob", "class")));
        map = TestSolrConfigHandlerConcurrent.getAsMap(baseUrl + "/.system/schema/fields/blob", cloudClient);
        assertNotNull(map);
        assertEquals("blob", getObjectByPath(map, true, Arrays.asList("field", "name")));
        assertEquals("bytes", getObjectByPath(map, true, Arrays.asList("field", "type")));
        checkBlobPost(baseUrl, cloudClient);
    }
}

Example 58 with Replica

use of org.apache.solr.common.cloud.Replica in project lucene-solr by apache.

the class TestConfigReload method checkConfReload.

private void checkConfReload(SolrZkClient client, String resPath, String name, String uri) throws Exception {
    Stat stat = new Stat();
    byte[] data = null;
    try {
        data = client.getData(resPath, null, stat, true);
    } catch (KeeperException.NoNodeException e) {
        data = "{}".getBytes(StandardCharsets.UTF_8);
        log.info("creating_node {}", resPath);
        client.create(resPath, data, CreateMode.PERSISTENT, true);
    }
    long startTime = System.nanoTime();
    Stat newStat = client.setData(resPath, data, true);
    client.setData("/configs/conf1", new byte[] { 1 }, true);
    assertTrue(newStat.getVersion() > stat.getVersion());
    log.info("new_version " + newStat.getVersion());
    Integer newVersion = newStat.getVersion();
    long maxTimeoutSeconds = 20;
    DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection("collection1");
    List<String> urls = new ArrayList<>();
    for (Slice slice : coll.getSlices()) {
        for (Replica replica : slice.getReplicas()) urls.add("" + replica.get(ZkStateReader.BASE_URL_PROP) + "/" + replica.get(ZkStateReader.CORE_NAME_PROP));
    }
    HashSet<String> succeeded = new HashSet<>();
    while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
        Thread.sleep(50);
        for (String url : urls) {
            Map respMap = getAsMap(url + uri + "?wt=json");
            if (String.valueOf(newVersion).equals(String.valueOf(getObjectByPath(respMap, true, asList(name, "znodeVersion"))))) {
                succeeded.add(url);
            }
        }
        if (succeeded.size() == urls.size())
            break;
        succeeded.clear();
    }
    assertEquals(StrUtils.formatString("tried these servers {0} succeeded only in {1} ", urls, succeeded), urls.size(), succeeded.size());
}

Example 59 with Replica

use of org.apache.solr.common.cloud.Replica in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method ensureAllReplicasAreActive.

protected List<Replica> ensureAllReplicasAreActive(String testCollectionName, String shardId, int shards, int rf, int maxWaitSecs) throws Exception {
    final RTimer timer = new RTimer();
    Map<String, Replica> notLeaders = new HashMap<>();
    ZkStateReader zkr = cloudClient.getZkStateReader();
    // force the state to be fresh
    zkr.forceUpdateCollection(testCollectionName);
    ClusterState cs = zkr.getClusterState();
    Collection<Slice> slices = cs.getActiveSlices(testCollectionName);
    assertTrue(slices.size() == shards);
    boolean allReplicasUp = false;
    long waitMs = 0L;
    long maxWaitMs = maxWaitSecs * 1000L;
    Replica leader = null;
    while (waitMs < maxWaitMs && !allReplicasUp) {
        cs = cloudClient.getZkStateReader().getClusterState();
        assertNotNull(cs);
        Slice shard = cs.getSlice(testCollectionName, shardId);
        assertNotNull("No Slice for " + shardId, shard);
        // assume true
        allReplicasUp = true;
        Collection<Replica> replicas = shard.getReplicas();
        assertTrue("Did not find correct number of replicas. Expected:" + rf + " Found:" + replicas.size(), replicas.size() == rf);
        leader = shard.getLeader();
        assertNotNull(leader);
        log.info("Found " + replicas.size() + " replicas and leader on " + leader.getNodeName() + " for " + shardId + " in " + testCollectionName);
        // ensure all replicas are "active" and identify the non-leader replica
        for (Replica replica : replicas) {
            if (replica.getState() != Replica.State.ACTIVE) {
                log.info("Replica {} is currently {}", replica.getName(), replica.getState());
                allReplicasUp = false;
            }
            if (!leader.equals(replica))
                notLeaders.put(replica.getName(), replica);
        }
        if (!allReplicasUp) {
            try {
                Thread.sleep(500L);
            } catch (Exception ignoreMe) {
            }
            waitMs += 500L;
        }
    }
    if (!allReplicasUp)
        fail("Didn't see all replicas for shard " + shardId + " in " + testCollectionName + " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo());
    if (notLeaders.isEmpty())
        fail("Didn't isolate any replicas that are not the leader! ClusterState: " + printClusterStateInfo());
    log.info("Took {} ms to see all replicas become active.", timer.getTime());
    List<Replica> replicas = new ArrayList<>();
    replicas.addAll(notLeaders.values());
    return replicas;
}

Example 60 with Replica

use of org.apache.solr.common.cloud.Replica in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method checkCollectionExpectations.

private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
    ClusterState clusterState = getCommonCloudSolrClient().getZkStateReader().getClusterState();
    int expectedSlices = numShardsNumReplicaList.get(0);
    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
    //          .getCollectionStates();
    if (clusterState.hasCollection(collectionName)) {
        Map<String, Slice> slices = clusterState.getCollection(collectionName).getSlicesMap();
        // did we find expectedSlices slices/shards?
        if (slices.size() != expectedSlices) {
            return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
        }
        int totalShards = 0;
        for (String sliceName : slices.keySet()) {
            for (Replica replica : slices.get(sliceName).getReplicas()) {
                if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
                    return "Shard " + replica.getName() + " created on node " + replica.getNodeName() + " not allowed to run shards for the created collection " + collectionName;
                }
            }
            totalShards += slices.get(sliceName).getReplicas().size();
        }
        if (totalShards != expectedTotalShards) {
            return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
        }
        return null;
    } else {
        return "Could not find new collection " + collectionName;
    }
}