Examples with ShardHandler org.apache.solr.handler.component.ShardHandler used on opensource projects

Example 1 with ShardHandler

use of org.apache.solr.handler.component.ShardHandler in project lucene-solr by apache.

the class CollectionsHandler method forceLeaderElection.

private static void forceLeaderElection(SolrQueryRequest req, CollectionsHandler handler) {
    ClusterState clusterState = handler.coreContainer.getZkController().getClusterState();
    String collectionName = req.getParams().required().get(COLLECTION_PROP);
    String sliceId = req.getParams().required().get(SHARD_ID_PROP);
    log.info("Force leader invoked, state: {}", clusterState);
    DocCollection collection = clusterState.getCollection(collectionName);
    Slice slice = collection.getSlice(sliceId);
    if (slice == null) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "No shard with name " + sliceId + " exists for collection " + collectionName);
    }
    try {
        // if an active replica is the leader, then all is fine already
        Replica leader = slice.getLeader();
        if (leader != null && leader.getState() == State.ACTIVE) {
            throw new SolrException(ErrorCode.SERVER_ERROR, "The shard already has an active leader. Force leader is not applicable. State: " + slice);
        }
        // Clear out any LIR state
        String lirPath = handler.coreContainer.getZkController().getLeaderInitiatedRecoveryZnodePath(collectionName, sliceId);
        if (handler.coreContainer.getZkController().getZkClient().exists(lirPath, true)) {
            StringBuilder sb = new StringBuilder();
            handler.coreContainer.getZkController().getZkClient().printLayout(lirPath, 4, sb);
            log.info("Cleaning out LIR data, which was: {}", sb);
            handler.coreContainer.getZkController().getZkClient().clean(lirPath);
        }
        // state to active.
        for (Replica rep : slice.getReplicas()) {
            if (clusterState.getLiveNodes().contains(rep.getNodeName())) {
                ShardHandler shardHandler = handler.coreContainer.getShardHandlerFactory().getShardHandler();
                ModifiableSolrParams params = new ModifiableSolrParams();
                params.set(CoreAdminParams.ACTION, CoreAdminAction.FORCEPREPAREFORLEADERSHIP.toString());
                params.set(CoreAdminParams.CORE, rep.getStr("core"));
                String nodeName = rep.getNodeName();
                OverseerCollectionMessageHandler.sendShardRequest(nodeName, params, shardHandler, null, null, CommonParams.CORES_HANDLER_PATH, // synchronous request
                handler.coreContainer.getZkController().getZkStateReader());
            }
        }
        // Wait till we have an active leader
        boolean success = false;
        for (int i = 0; i < 9; i++) {
            Thread.sleep(5000);
            clusterState = handler.coreContainer.getZkController().getClusterState();
            collection = clusterState.getCollection(collectionName);
            slice = collection.getSlice(sliceId);
            if (slice.getLeader() != null && slice.getLeader().getState() == State.ACTIVE) {
                success = true;
                break;
            }
            log.warn("Force leader attempt {}. Waiting 5 secs for an active leader. State of the slice: {}", (i + 1), slice);
        }
        if (success) {
            log.info("Successfully issued FORCELEADER command for collection: {}, shard: {}", collectionName, sliceId);
        } else {
            log.info("Couldn't successfully force leader, collection: {}, shard: {}. Cluster state: {}", collectionName, sliceId, clusterState);
        }
    } catch (SolrException e) {
        throw e;
    } catch (Exception e) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "Error executing FORCELEADER operation for collection: " + collectionName + " shard: " + sliceId, e);
    }
}

Example 2 with ShardHandler

use of org.apache.solr.handler.component.ShardHandler in project lucene-solr by apache.

the class BackupCmd method copyIndexFiles.

private void copyIndexFiles(URI backupPath, ZkNodeProps request, NamedList results) throws Exception {
    String collectionName = request.getStr(COLLECTION_PROP);
    String backupName = request.getStr(NAME);
    String asyncId = request.getStr(ASYNC);
    String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
    ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
    Map<String, String> requestMap = new HashMap<>();
    String commitName = request.getStr(CoreAdminParams.COMMIT_NAME);
    Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
    if (commitName != null) {
        SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient();
        snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
        if (!snapshotMeta.isPresent()) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " does not exist for collection " + collectionName);
        }
        if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
        }
    }
    log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName, backupPath);
    Collection<String> shardsToConsider = Collections.emptySet();
    if (snapshotMeta.isPresent()) {
        shardsToConsider = snapshotMeta.get().getShards();
    }
    for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
        Replica replica = null;
        if (snapshotMeta.isPresent()) {
            if (!shardsToConsider.contains(slice.getName())) {
                log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.", slice.getName(), collectionName, snapshotMeta.get().getName());
                continue;
            }
            replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
        } else {
            // Note - Actually this can return a null value when there is no leader for this shard.
            replica = slice.getLeader();
            if (replica == null) {
                throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
            }
        }
        String coreName = replica.getStr(CORE_NAME_PROP);
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
        params.set(NAME, slice.getName());
        params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
        // note: index dir will be here then the "snapshot." + slice name
        params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString());
        params.set(CORE_NAME_PROP, coreName);
        if (snapshotMeta.isPresent()) {
            params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
        }
        ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
        log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
    }
    log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);
    ocmh.processResponses(results, shardHandler, true, "Could not backup all replicas", asyncId, requestMap);
}

Example 3 with ShardHandler

use of org.apache.solr.handler.component.ShardHandler in project lucene-solr by apache.

the class CreateCollectionCmd method call.

@Override
public void call(ClusterState clusterState, ZkNodeProps message, NamedList results) throws Exception {
    final String collectionName = message.getStr(NAME);
    log.info("Create collection {}", collectionName);
    if (clusterState.hasCollection(collectionName)) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName);
    }
    String configName = getConfigName(collectionName, message);
    if (configName == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No config set found to associate with the collection.");
    }
    ocmh.validateConfigOrThrowSolrException(configName);
    try {
        // look at the replication factor and see if it matches reality
        // if it does not, find best nodes to create more cores
        int numTlogReplicas = message.getInt(TLOG_REPLICAS, 0);
        int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, numTlogReplicas > 0 ? 0 : 1));
        int numPullReplicas = message.getInt(PULL_REPLICAS, 0);
        ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
        final String async = message.getStr(ASYNC);
        Integer numSlices = message.getInt(NUM_SLICES, null);
        String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);
        List<String> shardNames = new ArrayList<>();
        if (ImplicitDocRouter.NAME.equals(router)) {
            ClusterStateMutator.getShardNames(shardNames, message.getStr("shards", null));
            numSlices = shardNames.size();
        } else {
            if (numSlices == null) {
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required param (when using CompositeId router).");
            }
            ClusterStateMutator.getShardNames(numSlices, shardNames);
        }
        int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
        if (numNrtReplicas + numTlogReplicas <= 0) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
        }
        if (numSlices <= 0) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0");
        }
        // we need to look at every node and see how many cores it serves
        // add our new cores to existing nodes serving the least number of cores
        // but (for now) require that each core goes on a distinct node.
        final List<String> nodeList = OverseerCollectionMessageHandler.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, RANDOM);
        Map<ReplicaAssigner.Position, String> positionVsNodes;
        if (nodeList.isEmpty()) {
            log.warn("It is unusual to create a collection (" + collectionName + ") without cores.");
            positionVsNodes = new HashMap<>();
        } else {
            int totalNumReplicas = numNrtReplicas + numTlogReplicas + numPullReplicas;
            if (totalNumReplicas > nodeList.size()) {
                log.warn("Specified number of replicas of " + totalNumReplicas + " on collection " + collectionName + " is higher than the number of Solr instances currently live or live and part of your " + CREATE_NODE_SET + "(" + nodeList.size() + "). It's unusual to run two replica of the same slice on the same Solr-instance.");
            }
            int maxShardsAllowedToCreate = maxShardsPerNode * nodeList.size();
            int requestedShardsToCreate = numSlices * totalNumReplicas;
            if (maxShardsAllowedToCreate < requestedShardsToCreate) {
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of " + MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode + ", and the number of nodes currently live or live and part of your " + CREATE_NODE_SET + " is " + nodeList.size() + ". This allows a maximum of " + maxShardsAllowedToCreate + " to be created. Value of " + NUM_SLICES + " is " + numSlices + ", value of " + NRT_REPLICAS + " is " + numNrtReplicas + ", value of " + TLOG_REPLICAS + " is " + numTlogReplicas + " and value of " + PULL_REPLICAS + " is " + numPullReplicas + ". This requires " + requestedShardsToCreate + " shards to be created (higher than the allowed number)");
            }
            positionVsNodes = ocmh.identifyNodes(clusterState, nodeList, message, shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas);
        }
        ZkStateReader zkStateReader = ocmh.zkStateReader;
        boolean isLegacyCloud = Overseer.isLegacy(zkStateReader);
        ocmh.createConfNode(configName, collectionName, isLegacyCloud);
        Map<String, String> collectionParams = new HashMap<>();
        Map<String, Object> collectionProps = message.getProperties();
        for (String propName : collectionProps.keySet()) {
            if (propName.startsWith(ZkController.COLLECTION_PARAM_PREFIX)) {
                collectionParams.put(propName.substring(ZkController.COLLECTION_PARAM_PREFIX.length()), (String) collectionProps.get(propName));
            }
        }
        createCollectionZkNode(zkClient, collectionName, collectionParams);
        Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message));
        // wait for a while until we don't see the collection
        TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS);
        boolean created = false;
        while (!waitUntil.hasTimedOut()) {
            Thread.sleep(100);
            created = zkStateReader.getClusterState().hasCollection(collectionName);
            if (created)
                break;
        }
        if (!created)
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Could not fully create collection: " + collectionName);
        if (nodeList.isEmpty()) {
            log.debug("Finished create command for collection: {}", collectionName);
            return;
        }
        // For tracking async calls.
        Map<String, String> requestMap = new HashMap<>();
        log.debug(formatString("Creating SolrCores for new collection {0}, shardNames {1} , nrtReplicas : {2}, tlogReplicas: {3}, pullReplicas: {4}", collectionName, shardNames, numNrtReplicas, numTlogReplicas, numPullReplicas));
        Map<String, ShardRequest> coresToCreate = new LinkedHashMap<>();
        for (Map.Entry<ReplicaAssigner.Position, String> e : positionVsNodes.entrySet()) {
            ReplicaAssigner.Position position = e.getKey();
            String nodeName = e.getValue();
            String coreName = Assign.buildCoreName(collectionName, position.shard, position.type, position.index + 1);
            log.debug(formatString("Creating core {0} as part of shard {1} of collection {2} on {3}", coreName, position.shard, collectionName, nodeName));
            String baseUrl = zkStateReader.getBaseUrlForNodeName(nodeName);
            // Otherwise the core creation fails
            if (!isLegacyCloud) {
                ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toString(), ZkStateReader.COLLECTION_PROP, collectionName, ZkStateReader.SHARD_ID_PROP, position.shard, ZkStateReader.CORE_NAME_PROP, coreName, ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), ZkStateReader.BASE_URL_PROP, baseUrl, ZkStateReader.REPLICA_TYPE, position.type.name());
                Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props));
            }
            // Need to create new params for each request
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString());
            params.set(CoreAdminParams.NAME, coreName);
            params.set(COLL_CONF, configName);
            params.set(CoreAdminParams.COLLECTION, collectionName);
            params.set(CoreAdminParams.SHARD, position.shard);
            params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices);
            params.set(CoreAdminParams.NEW_COLLECTION, "true");
            params.set(CoreAdminParams.REPLICA_TYPE, position.type.name());
            if (async != null) {
                String coreAdminAsyncId = async + Math.abs(System.nanoTime());
                params.add(ASYNC, coreAdminAsyncId);
                requestMap.put(nodeName, coreAdminAsyncId);
            }
            ocmh.addPropertyParams(message, params);
            ShardRequest sreq = new ShardRequest();
            sreq.nodeName = nodeName;
            params.set("qt", ocmh.adminPath);
            sreq.purpose = 1;
            sreq.shards = new String[] { baseUrl };
            sreq.actualShards = sreq.shards;
            sreq.params = params;
            if (isLegacyCloud) {
                shardHandler.submit(sreq, sreq.shards[0], sreq.params);
            } else {
                coresToCreate.put(coreName, sreq);
            }
        }
        if (!isLegacyCloud) {
            // wait for all replica entries to be created
            Map<String, Replica> replicas = ocmh.waitToSeeReplicasInState(collectionName, coresToCreate.keySet());
            for (Map.Entry<String, ShardRequest> e : coresToCreate.entrySet()) {
                ShardRequest sreq = e.getValue();
                sreq.params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName());
                shardHandler.submit(sreq, sreq.shards[0], sreq.params);
            }
        }
        ocmh.processResponses(results, shardHandler, false, null, async, requestMap, Collections.emptySet());
        if (results.get("failure") != null && ((SimpleOrderedMap) results.get("failure")).size() > 0) {
            // Let's cleanup as we hit an exception
            // We shouldn't be passing 'results' here for the cleanup as the response would then contain 'success'
            // element, which may be interpreted by the user as a positive ack
            ocmh.cleanupCollection(collectionName, new NamedList());
            log.info("Cleaned up artifacts for failed create collection for [{}]", collectionName);
        } else {
            log.debug("Finished create command on all shards for collection: {}", collectionName);
        }
    } catch (SolrException ex) {
        throw ex;
    } catch (Exception ex) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, ex);
    }
}

Example 4 with ShardHandler

use of org.apache.solr.handler.component.ShardHandler in project lucene-solr by apache.

the class DeleteSnapshotCmd method call.

@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
    String collectionName = message.getStr(COLLECTION_PROP);
    String commitName = message.getStr(CoreAdminParams.COMMIT_NAME);
    String asyncId = message.getStr(ASYNC);
    Map<String, String> requestMap = new HashMap<>();
    NamedList shardRequestResults = new NamedList();
    ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
    SolrZkClient zkClient = ocmh.overseer.getZkController().getZkClient();
    Optional<CollectionSnapshotMetaData> meta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
    if (!meta.isPresent()) {
        // Snapshot not found. Nothing to do.
        return;
    }
    log.info("Deleting a snapshot for collection={} with commitName={}", collectionName, commitName);
    Set<String> existingCores = new HashSet<>();
    for (Slice s : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getSlices()) {
        for (Replica r : s.getReplicas()) {
            existingCores.add(r.getCoreName());
        }
    }
    Set<String> coresWithSnapshot = new HashSet<>();
    for (CoreSnapshotMetaData m : meta.get().getReplicaSnapshots()) {
        if (existingCores.contains(m.getCoreName())) {
            coresWithSnapshot.add(m.getCoreName());
        }
    }
    log.info("Existing cores with snapshot for collection={} are {}", collectionName, existingCores);
    for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getSlices()) {
        for (Replica replica : slice.getReplicas()) {
            if (replica.getState() == State.DOWN) {
                // Since replica is down - no point sending a request.
                continue;
            }
            // replicas to contact at this point, we try on all replicas.
            if (meta.get().getStatus() == SnapshotStatus.InProgress || coresWithSnapshot.contains(replica.getCoreName())) {
                String coreName = replica.getStr(CORE_NAME_PROP);
                ModifiableSolrParams params = new ModifiableSolrParams();
                params.set(CoreAdminParams.ACTION, CoreAdminAction.DELETESNAPSHOT.toString());
                params.set(NAME, slice.getName());
                params.set(CORE_NAME_PROP, coreName);
                params.set(CoreAdminParams.COMMIT_NAME, commitName);
                log.info("Sending deletesnapshot request to core={} with commitName={}", coreName, commitName);
                ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
            }
        }
    }
    ocmh.processResponses(shardRequestResults, shardHandler, false, null, asyncId, requestMap);
    NamedList success = (NamedList) shardRequestResults.get("success");
    List<CoreSnapshotMetaData> replicas = new ArrayList<>();
    if (success != null) {
        for (int i = 0; i < success.size(); i++) {
            NamedList resp = (NamedList) success.getVal(i);
            // Unfortunately async processing logic doesn't provide the "core" name automatically.
            String coreName = (String) resp.get("core");
            coresWithSnapshot.remove(coreName);
        }
    }
    if (!coresWithSnapshot.isEmpty()) {
        // One or more failures.
        log.warn("Failed to delete a snapshot for collection {} with commitName = {}. Snapshot could not be deleted for following cores {}", collectionName, commitName, coresWithSnapshot);
        List<CoreSnapshotMetaData> replicasWithSnapshot = new ArrayList<>();
        for (CoreSnapshotMetaData m : meta.get().getReplicaSnapshots()) {
            if (coresWithSnapshot.contains(m.getCoreName())) {
                replicasWithSnapshot.add(m);
            }
        }
        // Update the ZK meta-data to include only cores with the snapshot. This will enable users to figure out
        // which cores still contain the named snapshot.
        CollectionSnapshotMetaData newResult = new CollectionSnapshotMetaData(meta.get().getName(), SnapshotStatus.Failed, meta.get().getCreationDate(), replicasWithSnapshot);
        SolrSnapshotManager.updateCollectionLevelSnapshot(zkClient, collectionName, newResult);
        log.info("Saved snapshot information for collection={} with commitName={} in Zookeeper as follows", collectionName, commitName, Utils.toJSON(newResult));
        throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to delete snapshot on cores " + coresWithSnapshot);
    } else {
        // Delete the ZK path so that we eliminate the references of this snapshot from collection level meta-data.
        SolrSnapshotManager.deleteCollectionLevelSnapshot(zkClient, collectionName, commitName);
        log.info("Deleted Zookeeper snapshot metdata for collection={} with commitName={}", collectionName, commitName);
        log.info("Successfully deleted snapshot for collection={} with commitName={}", collectionName, commitName);
    }
}

Example 5 with ShardHandler

use of org.apache.solr.handler.component.ShardHandler in project lucene-solr by apache.

the class AddReplicaCmd method addReplica.

ZkNodeProps addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete) throws KeeperException, InterruptedException {
    log.info("addReplica() : {}", Utils.toJSONString(message));
    String collection = message.getStr(COLLECTION_PROP);
    String node = message.getStr(CoreAdminParams.NODE);
    String shard = message.getStr(SHARD_ID_PROP);
    String coreName = message.getStr(CoreAdminParams.NAME);
    Replica.Type replicaType = Replica.Type.valueOf(message.getStr(ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.name()).toUpperCase(Locale.ROOT));
    boolean parallel = message.getBool("parallel", false);
    if (StringUtils.isBlank(coreName)) {
        coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
    }
    final String asyncId = message.getStr(ASYNC);
    DocCollection coll = clusterState.getCollection(collection);
    if (coll == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist");
    }
    if (coll.getSlice(shard) == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " shard: " + shard + " does not exist");
    }
    ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
    boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
    // Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place.
    if (!skipCreateReplicaInClusterState) {
        node = getNodesForNewReplicas(clusterState, collection, shard, 1, node, ocmh.overseer.getZkController().getCoreContainer()).get(0).nodeName;
    }
    log.info("Node Identified {} for creating new replica", node);
    if (!clusterState.liveNodesContain(node)) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
    }
    if (coreName == null) {
        coreName = Assign.buildCoreName(coll, shard, replicaType);
    } else if (!skipCreateReplicaInClusterState) {
        //Validate that the core name is unique in that collection
        for (Slice slice : coll.getSlices()) {
            for (Replica replica : slice.getReplicas()) {
                String replicaCoreName = replica.getStr(CORE_NAME_PROP);
                if (coreName.equals(replicaCoreName)) {
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" + " for this collection");
                }
            }
        }
    }
    ModifiableSolrParams params = new ModifiableSolrParams();
    ZkStateReader zkStateReader = ocmh.zkStateReader;
    if (!Overseer.isLegacy(zkStateReader)) {
        if (!skipCreateReplicaInClusterState) {
            ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), ZkStateReader.COLLECTION_PROP, collection, ZkStateReader.SHARD_ID_PROP, shard, ZkStateReader.CORE_NAME_PROP, coreName, ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), ZkStateReader.BASE_URL_PROP, zkStateReader.getBaseUrlForNodeName(node), ZkStateReader.NODE_NAME_PROP, node, ZkStateReader.REPLICA_TYPE, replicaType.name());
            Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props));
        }
        params.set(CoreAdminParams.CORE_NODE_NAME, ocmh.waitToSeeReplicasInState(collection, Collections.singletonList(coreName)).get(coreName).getName());
    }
    String configName = zkStateReader.readConfigName(collection);
    String routeKey = message.getStr(ShardParams._ROUTE_);
    String dataDir = message.getStr(CoreAdminParams.DATA_DIR);
    String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR);
    params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString());
    params.set(CoreAdminParams.NAME, coreName);
    params.set(COLL_CONF, configName);
    params.set(CoreAdminParams.COLLECTION, collection);
    params.set(CoreAdminParams.REPLICA_TYPE, replicaType.name());
    if (shard != null) {
        params.set(CoreAdminParams.SHARD, shard);
    } else if (routeKey != null) {
        Collection<Slice> slices = coll.getRouter().getSearchSlicesSingle(routeKey, null, coll);
        if (slices.isEmpty()) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No active shard serving _route_=" + routeKey + " found");
        } else {
            params.set(CoreAdminParams.SHARD, slices.iterator().next().getName());
        }
    } else {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Specify either 'shard' or _route_ param");
    }
    if (dataDir != null) {
        params.set(CoreAdminParams.DATA_DIR, dataDir);
    }
    if (instanceDir != null) {
        params.set(CoreAdminParams.INSTANCE_DIR, instanceDir);
    }
    ocmh.addPropertyParams(message, params);
    // For tracking async calls.
    Map<String, String> requestMap = new HashMap<>();
    ocmh.sendShardRequest(node, params, shardHandler, asyncId, requestMap);
    final String fnode = node;
    final String fcoreName = coreName;
    Runnable runnable = () -> {
        ocmh.processResponses(results, shardHandler, true, "ADDREPLICA failed to create replica", asyncId, requestMap);
        ocmh.waitForCoreNodeName(collection, fnode, fcoreName);
        if (onComplete != null)
            onComplete.run();
    };
    if (!parallel) {
        runnable.run();
    } else {
        ocmh.tpe.submit(runnable);
    }
    return new ZkNodeProps(ZkStateReader.COLLECTION_PROP, collection, ZkStateReader.SHARD_ID_PROP, shard, ZkStateReader.CORE_NAME_PROP, coreName, ZkStateReader.NODE_NAME_PROP, node);
}