Examples with DocCollection org.apache.solr.common.cloud.DocCollection used on opensource projects

Example 6 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class SliceMutator method addReplica.

public ZkWriteCommand addReplica(ClusterState clusterState, ZkNodeProps message) {
    log.info("createReplica() {} ", message);
    String coll = message.getStr(ZkStateReader.COLLECTION_PROP);
    if (!checkCollectionKeyExistence(message))
        return ZkStateWriter.NO_OP;
    String slice = message.getStr(ZkStateReader.SHARD_ID_PROP);
    DocCollection collection = clusterState.getCollection(coll);
    Slice sl = collection.getSlice(slice);
    if (sl == null) {
        log.error("Invalid Collection/Slice {}/{} ", coll, slice);
        return ZkStateWriter.NO_OP;
    }
    String coreNodeName = Assign.assignNode(collection);
    Replica replica = new Replica(coreNodeName, makeMap(ZkStateReader.CORE_NAME_PROP, message.getStr(ZkStateReader.CORE_NAME_PROP), ZkStateReader.BASE_URL_PROP, message.getStr(ZkStateReader.BASE_URL_PROP), ZkStateReader.STATE_PROP, message.getStr(ZkStateReader.STATE_PROP), ZkStateReader.NODE_NAME_PROP, message.getStr(ZkStateReader.NODE_NAME_PROP), ZkStateReader.REPLICA_TYPE, message.get(ZkStateReader.REPLICA_TYPE)));
    return new ZkWriteCommand(coll, updateReplica(collection, sl, replica.getName(), replica));
}

Example 7 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class SliceMutator method updateShardState.

public ZkWriteCommand updateShardState(ClusterState clusterState, ZkNodeProps message) {
    String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
    if (!checkCollectionKeyExistence(message))
        return ZkStateWriter.NO_OP;
    log.info("Update shard state invoked for collection: " + collectionName + " with message: " + message);
    DocCollection collection = clusterState.getCollection(collectionName);
    Map<String, Slice> slicesCopy = new LinkedHashMap<>(collection.getSlicesMap());
    for (String key : message.keySet()) {
        if (ZkStateReader.COLLECTION_PROP.equals(key))
            continue;
        if (Overseer.QUEUE_OPERATION.equals(key))
            continue;
        Slice slice = collection.getSlice(key);
        if (slice == null) {
            throw new RuntimeException("Overseer.updateShardState unknown collection: " + collectionName + " slice: " + key);
        }
        log.info("Update shard state " + key + " to " + message.getStr(key));
        Map<String, Object> props = slice.shallowCopy();
        if (Slice.State.getState(message.getStr(key)) == Slice.State.ACTIVE) {
            props.remove(Slice.PARENT);
            props.remove("shard_parent_node");
            props.remove("shard_parent_zk_session");
        }
        props.put(ZkStateReader.STATE_PROP, message.getStr(key));
        Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props);
        slicesCopy.put(slice.getName(), newSlice);
    }
    return new ZkWriteCommand(collectionName, collection.copyWithSlices(slicesCopy));
}

Example 8 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CloserThread method reload.

/**
   * Recreates a SolrCore.
   * While the new core is loading, requests will continue to be dispatched to
   * and processed by the old core
   * 
   * @param name the name of the SolrCore to reload
   */
public void reload(String name) {
    SolrCore core = solrCores.getCoreFromAnyList(name, false);
    if (core != null) {
        // The underlying core properties files may have changed, we don't really know. So we have a (perhaps) stale
        // CoreDescriptor and we need to reload it from the disk files
        CoreDescriptor cd = reloadCoreDescriptor(core.getCoreDescriptor());
        solrCores.addCoreDescriptor(cd);
        try {
            solrCores.waitAddPendingCoreOps(cd.getName());
            ConfigSet coreConfig = coreConfigService.getConfig(cd);
            log.info("Reloading SolrCore '{}' using configuration from {}", cd.getName(), coreConfig.getName());
            SolrCore newCore = core.reload(coreConfig);
            registerCore(cd, newCore, false, false);
            if (getZkController() != null) {
                DocCollection docCollection = getZkController().getClusterState().getCollection(cd.getCollectionName());
                Replica replica = docCollection.getReplica(cd.getCloudDescriptor().getCoreNodeName());
                assert replica != null;
                if (replica.getType() == Replica.Type.TLOG) {
                    //TODO: needed here?
                    getZkController().stopReplicationFromLeader(core.getName());
                    if (!cd.getCloudDescriptor().isLeader()) {
                        getZkController().startReplicationFromLeader(newCore.getName(), true);
                    }
                }
            }
        } catch (SolrCoreState.CoreIsClosedException e) {
            throw e;
        } catch (Exception e) {
            coreInitFailures.put(cd.getName(), new CoreLoadFailure(cd, e));
            throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to reload core [" + cd.getName() + "]", e);
        } finally {
            solrCores.removeFromPendingOps(cd.getName());
        }
    } else {
        CoreLoadFailure clf = coreInitFailures.get(name);
        if (clf != null) {
            create(clf.cd, true, false);
        } else {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No such core: " + name);
        }
    }
}

Example 9 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CollectionsHandler method forceLeaderElection.

private static void forceLeaderElection(SolrQueryRequest req, CollectionsHandler handler) {
    ClusterState clusterState = handler.coreContainer.getZkController().getClusterState();
    String collectionName = req.getParams().required().get(COLLECTION_PROP);
    String sliceId = req.getParams().required().get(SHARD_ID_PROP);
    log.info("Force leader invoked, state: {}", clusterState);
    DocCollection collection = clusterState.getCollection(collectionName);
    Slice slice = collection.getSlice(sliceId);
    if (slice == null) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "No shard with name " + sliceId + " exists for collection " + collectionName);
    }
    try {
        // if an active replica is the leader, then all is fine already
        Replica leader = slice.getLeader();
        if (leader != null && leader.getState() == State.ACTIVE) {
            throw new SolrException(ErrorCode.SERVER_ERROR, "The shard already has an active leader. Force leader is not applicable. State: " + slice);
        }
        // Clear out any LIR state
        String lirPath = handler.coreContainer.getZkController().getLeaderInitiatedRecoveryZnodePath(collectionName, sliceId);
        if (handler.coreContainer.getZkController().getZkClient().exists(lirPath, true)) {
            StringBuilder sb = new StringBuilder();
            handler.coreContainer.getZkController().getZkClient().printLayout(lirPath, 4, sb);
            log.info("Cleaning out LIR data, which was: {}", sb);
            handler.coreContainer.getZkController().getZkClient().clean(lirPath);
        }
        // state to active.
        for (Replica rep : slice.getReplicas()) {
            if (clusterState.getLiveNodes().contains(rep.getNodeName())) {
                ShardHandler shardHandler = handler.coreContainer.getShardHandlerFactory().getShardHandler();
                ModifiableSolrParams params = new ModifiableSolrParams();
                params.set(CoreAdminParams.ACTION, CoreAdminAction.FORCEPREPAREFORLEADERSHIP.toString());
                params.set(CoreAdminParams.CORE, rep.getStr("core"));
                String nodeName = rep.getNodeName();
                OverseerCollectionMessageHandler.sendShardRequest(nodeName, params, shardHandler, null, null, CommonParams.CORES_HANDLER_PATH, // synchronous request
                handler.coreContainer.getZkController().getZkStateReader());
            }
        }
        // Wait till we have an active leader
        boolean success = false;
        for (int i = 0; i < 9; i++) {
            Thread.sleep(5000);
            clusterState = handler.coreContainer.getZkController().getClusterState();
            collection = clusterState.getCollection(collectionName);
            slice = collection.getSlice(sliceId);
            if (slice.getLeader() != null && slice.getLeader().getState() == State.ACTIVE) {
                success = true;
                break;
            }
            log.warn("Force leader attempt {}. Waiting 5 secs for an active leader. State of the slice: {}", (i + 1), slice);
        }
        if (success) {
            log.info("Successfully issued FORCELEADER command for collection: {}, shard: {}", collectionName, sliceId);
        } else {
            log.info("Couldn't successfully force leader, collection: {}, shard: {}. Cluster state: {}", collectionName, sliceId, clusterState);
        }
    } catch (SolrException e) {
        throw e;
    } catch (Exception e) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "Error executing FORCELEADER operation for collection: " + collectionName + " shard: " + sliceId, e);
    }
}

Example 10 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class AbstractCloudBackupRestoreTestCase method testBackupAndRestore.

private void testBackupAndRestore(String collectionName) throws Exception {
    String backupLocation = getBackupLocation();
    String backupName = "mytestbackup";
    CloudSolrClient client = cluster.getSolrClient();
    DocCollection backupCollection = client.getZkStateReader().getClusterState().getCollection(collectionName);
    Map<String, Integer> origShardToDocCount = getShardToDocCountMap(client, backupCollection);
    assert origShardToDocCount.isEmpty() == false;
    log.info("Triggering Backup command");
    {
        CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName).setLocation(backupLocation).setRepositoryName(getBackupRepoName());
        if (random().nextBoolean()) {
            assertEquals(0, backup.process(client).getStatus());
        } else {
            //async
            assertEquals(RequestStatusState.COMPLETED, backup.processAndWait(client, 30));
        }
    }
    log.info("Triggering Restore command");
    String restoreCollectionName = collectionName + "_restored";
    boolean sameConfig = random().nextBoolean();
    {
        CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName).setLocation(backupLocation).setRepositoryName(getBackupRepoName());
        if (backupCollection.getReplicas().size() > cluster.getJettySolrRunners().size()) {
            // may need to increase maxShardsPerNode (e.g. if it was shard split, then now we need more)
            restore.setMaxShardsPerNode((int) Math.ceil(backupCollection.getReplicas().size() / cluster.getJettySolrRunners().size()));
        }
        if (rarely()) {
            // Try with createNodeSet configuration
            int nodeSetSize = cluster.getJettySolrRunners().size() / 2;
            List<String> nodeStrs = new ArrayList<>(nodeSetSize);
            Iterator<JettySolrRunner> iter = cluster.getJettySolrRunners().iterator();
            for (int i = 0; i < nodeSetSize; i++) {
                nodeStrs.add(iter.next().getNodeName());
            }
            restore.setCreateNodeSet(String.join(",", nodeStrs));
            restore.setCreateNodeSetShuffle(usually());
            // we need to double maxShardsPerNode value since we reduced number of available nodes by half.
            if (restore.getMaxShardsPerNode() != null) {
                restore.setMaxShardsPerNode(restore.getMaxShardsPerNode() * 2);
            } else {
                restore.setMaxShardsPerNode(origShardToDocCount.size() * 2);
            }
        }
        Properties props = new Properties();
        props.setProperty("customKey", "customVal");
        restore.setProperties(props);
        if (sameConfig == false) {
            restore.setConfigName("customConfigName");
        }
        if (random().nextBoolean()) {
            assertEquals(0, restore.process(client).getStatus());
        } else {
            //async
            assertEquals(RequestStatusState.COMPLETED, restore.processAndWait(client, 30));
        }
        AbstractDistribZkTestBase.waitForRecoveriesToFinish(restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
    }
    //Check the number of results are the same
    DocCollection restoreCollection = client.getZkStateReader().getClusterState().getCollection(restoreCollectionName);
    assertEquals(origShardToDocCount, getShardToDocCountMap(client, restoreCollection));
    //  test we reconstituted the hash ranges / doc router.
    if (!(restoreCollection.getRouter() instanceof ImplicitDocRouter) && random().nextBoolean()) {
        indexDocs(restoreCollectionName);
        assertEquals(origShardToDocCount, getShardToDocCountMap(client, restoreCollection));
    }
    assertEquals(backupCollection.getReplicationFactor(), restoreCollection.getReplicationFactor());
    assertEquals(backupCollection.getAutoAddReplicas(), restoreCollection.getAutoAddReplicas());
    assertEquals(backupCollection.getActiveSlices().iterator().next().getReplicas().size(), restoreCollection.getActiveSlices().iterator().next().getReplicas().size());
    assertEquals(sameConfig ? "conf1" : "customConfigName", cluster.getSolrClient().getZkStateReader().readConfigName(restoreCollectionName));
    Map<String, Integer> numReplicasByNodeName = new HashMap<>();
    restoreCollection.getReplicas().forEach(x -> {
        numReplicasByNodeName.put(x.getNodeName(), numReplicasByNodeName.getOrDefault(x.getNodeName(), 0) + 1);
    });
    numReplicasByNodeName.forEach((k, v) -> {
        assertTrue("Node " + k + " has " + v + " replicas. Expected num replicas : " + restoreCollection.getMaxShardsPerNode(), v <= restoreCollection.getMaxShardsPerNode());
    });
// assert added core properties:
// DWS: did via manual inspection.
// TODO Find the applicable core.properties on the file system but how?
}