Examples with DocCollection org.apache.solr.common.cloud.DocCollection used on opensource projects

Example 21 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ZkController method preRegister.

public void preRegister(CoreDescriptor cd) {
    String coreNodeName = getCoreNodeName(cd);
    // this also gets us our assigned shard id if it was not specified
    try {
        checkStateInZk(cd);
        CloudDescriptor cloudDesc = cd.getCloudDescriptor();
        // make sure the node name is set on the descriptor
        if (cloudDesc.getCoreNodeName() == null) {
            cloudDesc.setCoreNodeName(coreNodeName);
        }
        publish(cd, Replica.State.DOWN, false, true);
        String collectionName = cd.getCloudDescriptor().getCollectionName();
        DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
        log.debug(collection == null ? "Collection {} not visible yet, but flagging it so a watch is registered when it becomes visible" : "Registering watch for collection {}", collectionName);
        zkStateReader.registerCore(collectionName);
    } catch (KeeperException e) {
        log.error("", e);
        throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        log.error("", e);
        throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
    }
    if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getClusterState(), coreNodeName)) {
        doGetShardIdAndNodeNameProcess(cd);
    } else {
        // still wait till we see us in local state
        doGetShardIdAndNodeNameProcess(cd);
    }
}

Example 22 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class NodeMutator method downNode.

public List<ZkWriteCommand> downNode(ClusterState clusterState, ZkNodeProps message) {
    List<ZkWriteCommand> zkWriteCommands = new ArrayList<>();
    String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);
    log.debug("DownNode state invoked for node: " + nodeName);
    Map<String, DocCollection> collections = clusterState.getCollectionsMap();
    for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
        String collection = entry.getKey();
        DocCollection docCollection = entry.getValue();
        Map<String, Slice> slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap());
        boolean needToUpdateCollection = false;
        for (Entry<String, Slice> sliceEntry : slicesCopy.entrySet()) {
            Slice slice = sliceEntry.getValue();
            Map<String, Replica> newReplicas = slice.getReplicasCopy();
            Collection<Replica> replicas = slice.getReplicas();
            for (Replica replica : replicas) {
                String rNodeName = replica.getNodeName();
                if (rNodeName.equals(nodeName)) {
                    log.debug("Update replica state for " + replica + " to " + Replica.State.DOWN.toString());
                    Map<String, Object> props = replica.shallowCopy();
                    props.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
                    Replica newReplica = new Replica(replica.getName(), props);
                    newReplicas.put(replica.getName(), newReplica);
                    needToUpdateCollection = true;
                }
            }
            Slice newSlice = new Slice(slice.getName(), newReplicas, slice.shallowCopy());
            slicesCopy.put(slice.getName(), newSlice);
        }
        if (needToUpdateCollection) {
            zkWriteCommands.add(new ZkWriteCommand(collection, docCollection.copyWithSlices(slicesCopy)));
        }
    }
    return zkWriteCommands;
}

Example 23 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ReplicaMutator method addReplicaProperty.

public ZkWriteCommand addReplicaProperty(ClusterState clusterState, ZkNodeProps message) {
    if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false || checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false || checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false || checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false || checkKeyExistence(message, ZkStateReader.PROPERTY_VALUE_PROP) == false) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Overseer ADDREPLICAPROP requires " + ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " + ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " and " + ZkStateReader.PROPERTY_VALUE_PROP + " no action taken.");
    }
    String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
    String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
    String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
    String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
    if (StringUtils.startsWith(property, COLL_PROP_PREFIX) == false) {
        property = OverseerCollectionMessageHandler.COLL_PROP_PREFIX + property;
    }
    property = property.toLowerCase(Locale.ROOT);
    String propVal = message.getStr(ZkStateReader.PROPERTY_VALUE_PROP);
    String shardUnique = message.getStr(OverseerCollectionMessageHandler.SHARD_UNIQUE);
    boolean isUnique = false;
    if (SliceMutator.SLICE_UNIQUE_BOOLEAN_PROPERTIES.contains(property)) {
        if (StringUtils.isNotBlank(shardUnique) && Boolean.parseBoolean(shardUnique) == false) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Overseer ADDREPLICAPROP for " + property + " cannot have " + OverseerCollectionMessageHandler.SHARD_UNIQUE + " set to anything other than" + "'true'. No action taken");
        }
        isUnique = true;
    } else {
        isUnique = Boolean.parseBoolean(shardUnique);
    }
    DocCollection collection = clusterState.getCollection(collectionName);
    Replica replica = collection.getReplica(replicaName);
    if (replica == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " + collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
    }
    log.info("Setting property {} with value {} for collection {}", property, propVal, collectionName);
    log.debug("Full message: {}", message);
    // already the value we're going to set
    if (StringUtils.equalsIgnoreCase(replica.getStr(property), propVal))
        return ZkStateWriter.NO_OP;
    // OK, there's no way we won't change the cluster state now
    Map<String, Replica> replicas = collection.getSlice(sliceName).getReplicasCopy();
    if (isUnique == false) {
        replicas.get(replicaName).getProperties().put(property, propVal);
    } else {
        // Set prop for this replica, but remove it for all others.
        for (Replica rep : replicas.values()) {
            if (rep.getName().equalsIgnoreCase(replicaName)) {
                rep.getProperties().put(property, propVal);
            } else {
                rep.getProperties().remove(property);
            }
        }
    }
    Slice newSlice = new Slice(sliceName, replicas, collection.getSlice(sliceName).shallowCopy());
    DocCollection newCollection = CollectionMutator.updateSlice(collectionName, collection, newSlice);
    return new ZkWriteCommand(collectionName, newCollection);
}

Example 24 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ReplicaMutator method deleteReplicaProperty.

public ZkWriteCommand deleteReplicaProperty(ClusterState clusterState, ZkNodeProps message) {
    if (checkKeyExistence(message, ZkStateReader.COLLECTION_PROP) == false || checkKeyExistence(message, ZkStateReader.SHARD_ID_PROP) == false || checkKeyExistence(message, ZkStateReader.REPLICA_PROP) == false || checkKeyExistence(message, ZkStateReader.PROPERTY_PROP) == false) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Overseer DELETEREPLICAPROP requires " + ZkStateReader.COLLECTION_PROP + " and " + ZkStateReader.SHARD_ID_PROP + " and " + ZkStateReader.REPLICA_PROP + " and " + ZkStateReader.PROPERTY_PROP + " no action taken.");
    }
    String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
    String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
    String replicaName = message.getStr(ZkStateReader.REPLICA_PROP);
    String property = message.getStr(ZkStateReader.PROPERTY_PROP).toLowerCase(Locale.ROOT);
    if (StringUtils.startsWith(property, COLL_PROP_PREFIX) == false) {
        property = OverseerCollectionMessageHandler.COLL_PROP_PREFIX + property;
    }
    DocCollection collection = clusterState.getCollection(collectionName);
    Replica replica = collection.getReplica(replicaName);
    if (replica == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection/slice/replica " + collectionName + "/" + sliceName + "/" + replicaName + " no action taken.");
    }
    log.info("Deleting property {} for collection: {} slice: {} replica: {}", property, collectionName, sliceName, replicaName);
    log.debug("Full message: {}", message);
    String curProp = replica.getStr(property);
    // not there anyway, nothing to do.
    if (curProp == null)
        return ZkStateWriter.NO_OP;
    Slice slice = collection.getSlice(sliceName);
    DocCollection newCollection = SliceMutator.updateReplica(collection, slice, replicaName, unsetProperty(replica, property));
    return new ZkWriteCommand(collectionName, newCollection);
}

Example 25 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ReplicaMutator method updateState.

private ZkWriteCommand updateState(final ClusterState prevState, ZkNodeProps message, String collectionName, Integer numShards, boolean collectionExists) {
    String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
    String coreNodeName = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
    DocCollection collection = prevState.getCollectionOrNull(collectionName);
    if (coreNodeName == null) {
        coreNodeName = ClusterStateMutator.getAssignedCoreNodeName(collection, message.getStr(ZkStateReader.NODE_NAME_PROP), message.getStr(ZkStateReader.CORE_NAME_PROP));
        if (coreNodeName != null) {
            log.debug("node=" + coreNodeName + " is already registered");
        } else {
            // if coreNodeName is null, auto assign one
            coreNodeName = Assign.assignNode(collection);
        }
        message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
    }
    // use the provided non null shardId
    if (sliceName == null) {
        //get shardId from ClusterState
        sliceName = ClusterStateMutator.getAssignedId(collection, coreNodeName);
        if (sliceName != null) {
            log.debug("shard=" + sliceName + " is already registered");
        }
    }
    if (sliceName == null) {
        //request new shardId
        if (collectionExists) {
            // use existing numShards
            numShards = collection.getSlices().size();
            log.debug("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards);
        }
        sliceName = Assign.assignShard(collection, numShards);
        log.info("Assigning new node to shard shard=" + sliceName);
    }
    Slice slice = collection != null ? collection.getSlice(sliceName) : null;
    Map<String, Object> replicaProps = new LinkedHashMap<>();
    replicaProps.putAll(message.getProperties());
    if (slice != null) {
        Replica oldReplica = slice.getReplica(coreNodeName);
        if (oldReplica != null) {
            if (oldReplica.containsKey(ZkStateReader.LEADER_PROP)) {
                replicaProps.put(ZkStateReader.LEADER_PROP, oldReplica.get(ZkStateReader.LEADER_PROP));
            }
            replicaProps.put(ZkStateReader.REPLICA_TYPE, oldReplica.getType().toString());
            // Move custom props over.
            for (Map.Entry<String, Object> ent : oldReplica.getProperties().entrySet()) {
                if (ent.getKey().startsWith(COLL_PROP_PREFIX)) {
                    replicaProps.put(ent.getKey(), ent.getValue());
                }
            }
        }
    }
    // we don't put these in the clusterstate
    replicaProps.remove(ZkStateReader.NUM_SHARDS_PROP);
    replicaProps.remove(ZkStateReader.CORE_NODE_NAME_PROP);
    replicaProps.remove(ZkStateReader.SHARD_ID_PROP);
    replicaProps.remove(ZkStateReader.COLLECTION_PROP);
    replicaProps.remove(Overseer.QUEUE_OPERATION);
    // remove any props with null values
    Set<Map.Entry<String, Object>> entrySet = replicaProps.entrySet();
    List<String> removeKeys = new ArrayList<>();
    for (Map.Entry<String, Object> entry : entrySet) {
        if (entry.getValue() == null) {
            removeKeys.add(entry.getKey());
        }
    }
    for (String removeKey : removeKeys) {
        replicaProps.remove(removeKey);
    }
    replicaProps.remove(ZkStateReader.CORE_NODE_NAME_PROP);
    // remove shard specific properties
    String shardRange = (String) replicaProps.remove(ZkStateReader.SHARD_RANGE_PROP);
    String shardState = (String) replicaProps.remove(ZkStateReader.SHARD_STATE_PROP);
    String shardParent = (String) replicaProps.remove(ZkStateReader.SHARD_PARENT_PROP);
    Replica replica = new Replica(coreNodeName, replicaProps);
    log.debug("Will update state for replica: {}", replica);
    Map<String, Object> sliceProps = null;
    Map<String, Replica> replicas;
    if (slice != null) {
        collection = checkAndCompleteShardSplit(prevState, collection, coreNodeName, sliceName, replica);
        // get the current slice again because it may have been updated due to checkAndCompleteShardSplit method
        slice = collection.getSlice(sliceName);
        sliceProps = slice.getProperties();
        replicas = slice.getReplicasCopy();
    } else {
        replicas = new HashMap<>(1);
        sliceProps = new HashMap<>();
        sliceProps.put(Slice.RANGE, shardRange);
        sliceProps.put(ZkStateReader.STATE_PROP, shardState);
        sliceProps.put(Slice.PARENT, shardParent);
    }
    replicas.put(replica.getName(), replica);
    slice = new Slice(sliceName, replicas, sliceProps);
    DocCollection newCollection = CollectionMutator.updateSlice(collectionName, collection, slice);
    log.debug("Collection is now: {}", newCollection);
    return new ZkWriteCommand(collectionName, newCollection);
}