Examples with CoreContainer org.apache.solr.core.CoreContainer used on opensource projects

Example 86 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class TestLeaderInitiatedRecoveryThread method testPublishDownState.

public void testPublishDownState() throws Exception {
    waitForRecoveriesToFinish(true);
    final String leaderCoreNodeName = shardToLeaderJetty.get(SHARD1).coreNodeName;
    final CloudJettyRunner leaderRunner = shardToLeaderJetty.get(SHARD1);
    CoreContainer coreContainer = leaderRunner.jetty.getCoreContainer();
    ZkController zkController = coreContainer.getZkController();
    CloudJettyRunner notLeader = null;
    for (CloudJettyRunner cloudJettyRunner : shardToJetty.get(SHARD1)) {
        if (cloudJettyRunner != leaderRunner) {
            notLeader = cloudJettyRunner;
            break;
        }
    }
    assertNotNull(notLeader);
    Replica replica = cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, notLeader.coreNodeName);
    ZkCoreNodeProps replicaCoreNodeProps = new ZkCoreNodeProps(replica);
    MockCoreDescriptor cd = new MockCoreDescriptor() {

        public CloudDescriptor getCloudDescriptor() {
            return new CloudDescriptor(shardToLeaderJetty.get(SHARD1).info.getStr(ZkStateReader.CORE_NAME_PROP), new Properties(), this) {

                @Override
                public String getCoreNodeName() {
                    return shardToLeaderJetty.get(SHARD1).info.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
                }

                @Override
                public boolean isLeader() {
                    return true;
                }
            };
        }
    };
    /*
     1. Test that publishDownState throws exception when zkController.isReplicaInRecoveryHandling == false
      */
    try {
        LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd);
        assertFalse(zkController.isReplicaInRecoveryHandling(replicaCoreNodeProps.getCoreUrl()));
        thread.run();
        fail("publishDownState should not have succeeded because replica url is not marked in leader initiated recovery in ZkController");
    } catch (SolrException e) {
        assertTrue(e.code() == SolrException.ErrorCode.INVALID_STATE.code);
    }
    /*
     2. Test that a non-live replica cannot be put into LIR or down state
      */
    LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd);
    // kill the replica
    int children = cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size();
    ChaosMonkey.stop(notLeader.jetty);
    TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS);
    while (!timeOut.hasTimedOut()) {
        if (children > cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size()) {
            break;
        }
        Thread.sleep(500);
    }
    assertTrue(children > cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size());
    int cversion = getOverseerCversion();
    // Thread should not publish LIR and down state for node which is not live, regardless of whether forcePublish is true or false
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    // lets assert that we did not publish anything to overseer queue, simplest way is to assert that cversion of overseer queue zk node is still the same
    assertEquals(cversion, getOverseerCversion());
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    // lets assert that we did not publish anything to overseer queue
    assertEquals(cversion, getOverseerCversion());
    /*
    3. Test that if ZK connection loss then thread should not attempt to publish down state even if forcePublish=true
     */
    ChaosMonkey.start(notLeader.jetty);
    waitForRecoveriesToFinish(true);
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", new KeeperException.ConnectionLossException());
        }
    };
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    /*
     4. Test that if ZK connection loss or session expired then thread should not attempt to publish down state even if forcePublish=true
      */
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", new KeeperException.SessionExpiredException());
        }
    };
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    /*
     5. Test that any exception other then ZK connection loss or session expired should publish down state only if forcePublish=true
      */
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "bogus exception");
        }
    };
    // the following should return true because regardless of the bogus exception in setting LIR state, we still want recovery commands to be sent,
    // however the following will not publish a down state
    cversion = getOverseerCversion();
    assertTrue(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    // lets assert that we did not publish anything to overseer queue, simplest way is to assert that cversion of overseer queue zk node is still the same
    assertEquals(cversion, getOverseerCversion());
    assertTrue(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
    // this should have published a down state so assert that cversion has incremented
    assertTrue(getOverseerCversion() > cversion);
    timeOut = new TimeOut(30, TimeUnit.SECONDS);
    while (!timeOut.hasTimedOut()) {
        Replica r = cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, replica.getName());
        if (r.getState() == Replica.State.DOWN) {
            break;
        }
        Thread.sleep(500);
    }
    assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    assertEquals(Replica.State.DOWN, cloudClient.getZkStateReader().getClusterState().getReplica(DEFAULT_COLLECTION, replica.getName()).getState());
    /*
    6. Test that non-leader cannot set LIR nodes
     */
    coreContainer = notLeader.jetty.getCoreContainer();
    zkController = coreContainer.getZkController();
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer.getCores().iterator().next().getCoreDescriptor()) {

        @Override
        protected void updateLIRState(String replicaCoreNodeName) {
            try {
                super.updateLIRState(replicaCoreNodeName);
            } catch (Exception e) {
                assertTrue(e instanceof ZkController.NotLeaderException);
                throw e;
            }
        }
    };
    cversion = getOverseerCversion();
    assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
    assertEquals(cversion, getOverseerCversion());
    /*
     7. assert that we can write a LIR state if everything else is fine
      */
    // reset the zkcontroller to the one from the leader
    coreContainer = leaderRunner.jetty.getCoreContainer();
    zkController = coreContainer.getZkController();
    thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer, DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer.getCores().iterator().next().getCoreDescriptor());
    thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false);
    timeOut = new TimeOut(30, TimeUnit.SECONDS);
    while (!timeOut.hasTimedOut()) {
        Replica.State state = zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName());
        if (state == Replica.State.DOWN) {
            break;
        }
        Thread.sleep(500);
    }
    assertNotNull(zkController.getLeaderInitiatedRecoveryStateObject(DEFAULT_COLLECTION, SHARD1, replica.getName()));
    assertEquals(Replica.State.DOWN, zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
/*
    7. Test that
     */
}

Example 87 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class TestOnReconnectListenerSupport method test.

@Test
public void test() throws Exception {
    waitForThingsToLevelOut(30000);
    String testCollectionName = "c8n_onreconnect_1x1";
    String shardId = "shard1";
    createCollectionRetry(testCollectionName, 1, 1, 1);
    cloudClient.setDefaultCollection(testCollectionName);
    Replica leader = getShardLeader(testCollectionName, shardId, 30);
    JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(leader));
    // get the ZkController for the node hosting the leader
    CoreContainer cores = leaderJetty.getCoreContainer();
    ZkController zkController = cores.getZkController();
    assertNotNull("ZkController is null", zkController);
    String leaderCoreName = leader.getStr(CORE_NAME_PROP);
    String leaderCoreId;
    try (SolrCore leaderCore = cores.getCore(leaderCoreName)) {
        assertNotNull("SolrCore for " + leaderCoreName + " not found!", leaderCore);
        leaderCoreId = leaderCore.getName() + ":" + leaderCore.getStartNanoTime();
    }
    // verify the ZkIndexSchemaReader is a registered OnReconnect listener
    Set<OnReconnect> listeners = zkController.getCurrentOnReconnectListeners();
    assertNotNull("ZkController returned null OnReconnect listeners", listeners);
    ZkIndexSchemaReader expectedListener = null;
    for (OnReconnect listener : listeners) {
        if (listener instanceof ZkIndexSchemaReader) {
            ZkIndexSchemaReader reader = (ZkIndexSchemaReader) listener;
            if (leaderCoreId.equals(reader.getUniqueCoreId())) {
                expectedListener = reader;
                break;
            }
        }
    }
    assertNotNull("ZkIndexSchemaReader for core " + leaderCoreName + " not registered as an OnReconnect listener and should be", expectedListener);
    // reload the collection
    boolean wasReloaded = reloadCollection(leader, testCollectionName);
    assertTrue("Collection '" + testCollectionName + "' failed to reload within a reasonable amount of time!", wasReloaded);
    // after reload, the new core should be registered as an OnReconnect listener and the old should not be
    String reloadedLeaderCoreId;
    try (SolrCore leaderCore = cores.getCore(leaderCoreName)) {
        reloadedLeaderCoreId = leaderCore.getName() + ":" + leaderCore.getStartNanoTime();
    }
    // they shouldn't be equal after reload
    assertTrue(!leaderCoreId.equals(reloadedLeaderCoreId));
    listeners = zkController.getCurrentOnReconnectListeners();
    assertNotNull("ZkController returned null OnReconnect listeners", listeners);
    // reset
    expectedListener = null;
    for (OnReconnect listener : listeners) {
        if (listener instanceof ZkIndexSchemaReader) {
            ZkIndexSchemaReader reader = (ZkIndexSchemaReader) listener;
            if (leaderCoreId.equals(reader.getUniqueCoreId())) {
                fail("Previous core " + leaderCoreId + " should no longer be a registered OnReconnect listener! Current listeners: " + listeners);
            } else if (reloadedLeaderCoreId.equals(reader.getUniqueCoreId())) {
                expectedListener = reader;
                break;
            }
        }
    }
    assertNotNull("ZkIndexSchemaReader for core " + reloadedLeaderCoreId + " not registered as an OnReconnect listener and should be", expectedListener);
    // try to clean up
    try {
        CollectionAdminRequest.deleteCollection(testCollectionName).process(cloudClient);
    } catch (Exception e) {
        // don't fail the test
        log.warn("Could not delete collection {} after test completed", testCollectionName);
    }
    listeners = zkController.getCurrentOnReconnectListeners();
    for (OnReconnect listener : listeners) {
        if (listener instanceof ZkIndexSchemaReader) {
            ZkIndexSchemaReader reader = (ZkIndexSchemaReader) listener;
            if (reloadedLeaderCoreId.equals(reader.getUniqueCoreId())) {
                fail("Previous core " + reloadedLeaderCoreId + " should no longer be a registered OnReconnect listener after collection delete!");
            }
        }
    }
    log.info("TestOnReconnectListenerSupport succeeded ... shutting down now!");
}

Example 88 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class RestoreCmd method call.

@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
    // TODO maybe we can inherit createCollection's options/code
    String restoreCollectionName = message.getStr(COLLECTION_PROP);
    // of backup
    String backupName = message.getStr(NAME);
    ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
    String asyncId = message.getStr(ASYNC);
    String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
    Map<String, String> requestMap = new HashMap<>();
    CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer();
    BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo));
    URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
    URI backupPath = repository.resolve(location, backupName);
    ZkStateReader zkStateReader = ocmh.zkStateReader;
    BackupManager backupMgr = new BackupManager(repository, zkStateReader);
    Properties properties = backupMgr.readBackupProperties(location, backupName);
    String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP);
    DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection);
    // Get the Solr nodes to restore a collection.
    final List<String> nodeList = OverseerCollectionMessageHandler.getLiveOrLiveAndCreateNodeSetList(zkStateReader.getClusterState().getLiveNodes(), message, RANDOM);
    int numShards = backupCollectionState.getActiveSlices().size();
    int numNrtReplicas = getInt(message, NRT_REPLICAS, backupCollectionState.getNumNrtReplicas(), 0);
    if (numNrtReplicas == 0) {
        numNrtReplicas = getInt(message, REPLICATION_FACTOR, backupCollectionState.getReplicationFactor(), 0);
    }
    int numTlogReplicas = getInt(message, TLOG_REPLICAS, backupCollectionState.getNumTlogReplicas(), 0);
    int numPullReplicas = getInt(message, PULL_REPLICAS, backupCollectionState.getNumPullReplicas(), 0);
    int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas;
    int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, backupCollectionState.getMaxShardsPerNode());
    int availableNodeCount = nodeList.size();
    if ((numShards * totalReplicasPerShard) > (availableNodeCount * maxShardsPerNode)) {
        throw new SolrException(ErrorCode.BAD_REQUEST, String.format(Locale.ROOT, "Solr cloud with available number of nodes:%d is insufficient for" + " restoring a collection with %d shards, total replicas per shard %d and maxShardsPerNode %d." + " Consider increasing maxShardsPerNode value OR number of available nodes.", availableNodeCount, numShards, totalReplicasPerShard, maxShardsPerNode));
    }
    //Upload the configs
    String configName = (String) properties.get(COLL_CONF);
    String restoreConfigName = message.getStr(COLL_CONF, configName);
    if (zkStateReader.getConfigManager().configExists(restoreConfigName)) {
        log.info("Using existing config {}", restoreConfigName);
    //TODO add overwrite option?
    } else {
        log.info("Uploading config {}", restoreConfigName);
        backupMgr.uploadConfigDir(location, backupName, configName, restoreConfigName);
    }
    log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName, location);
    //Create core-less collection
    {
        Map<String, Object> propMap = new HashMap<>();
        propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString());
        // mostly true.  Prevents autoCreated=true in the collection state.
        propMap.put("fromApi", "true");
        // inherit settings from input API, defaulting to the backup's setting.  Ex: replicationFactor
        for (String collProp : COLL_PROPS.keySet()) {
            Object val = message.getProperties().getOrDefault(collProp, backupCollectionState.get(collProp));
            if (val != null) {
                propMap.put(collProp, val);
            }
        }
        propMap.put(NAME, restoreCollectionName);
        //no cores
        propMap.put(CREATE_NODE_SET, CREATE_NODE_SET_EMPTY);
        propMap.put(COLL_CONF, restoreConfigName);
        // router.*
        @SuppressWarnings("unchecked") Map<String, Object> routerProps = (Map<String, Object>) backupCollectionState.getProperties().get(DocCollection.DOC_ROUTER);
        for (Map.Entry<String, Object> pair : routerProps.entrySet()) {
            propMap.put(DocCollection.DOC_ROUTER + "." + pair.getKey(), pair.getValue());
        }
        Set<String> sliceNames = backupCollectionState.getActiveSlicesMap().keySet();
        if (backupCollectionState.getRouter() instanceof ImplicitDocRouter) {
            propMap.put(SHARDS_PROP, StrUtils.join(sliceNames, ','));
        } else {
            propMap.put(NUM_SLICES, sliceNames.size());
            // ClusterStateMutator.createCollection detects that "slices" is in fact a slice structure instead of a
            //   list of names, and if so uses this instead of building it.  We clear the replica list.
            Collection<Slice> backupSlices = backupCollectionState.getActiveSlices();
            Map<String, Slice> newSlices = new LinkedHashMap<>(backupSlices.size());
            for (Slice backupSlice : backupSlices) {
                newSlices.put(backupSlice.getName(), new Slice(backupSlice.getName(), Collections.emptyMap(), backupSlice.getProperties()));
            }
            propMap.put(SHARDS_PROP, newSlices);
        }
        ocmh.commandMap.get(CREATE).call(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList());
    // note: when createCollection() returns, the collection exists (no race)
    }
    DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
    DistributedQueue inQueue = Overseer.getStateUpdateQueue(zkStateReader.getZkClient());
    //Mark all shards in CONSTRUCTION STATE while we restore the data
    {
        //TODO might instead createCollection accept an initial state?  Is there a race?
        Map<String, Object> propMap = new HashMap<>();
        propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
        for (Slice shard : restoreCollection.getSlices()) {
            propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString());
        }
        propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
        inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
    }
    // TODO how do we leverage the RULE / SNITCH logic in createCollection?
    ClusterState clusterState = zkStateReader.getClusterState();
    List<String> sliceNames = new ArrayList<>();
    restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName()));
    Map<ReplicaAssigner.Position, String> positionVsNodes = ocmh.identifyNodes(clusterState, nodeList, message, sliceNames, numNrtReplicas, numTlogReplicas, numPullReplicas);
    //Create one replica per shard and copy backed up data to it
    for (Slice slice : restoreCollection.getSlices()) {
        log.debug("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
        HashMap<String, Object> propMap = new HashMap<>();
        propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD);
        propMap.put(COLLECTION_PROP, restoreCollectionName);
        propMap.put(SHARD_ID_PROP, slice.getName());
        if (numNrtReplicas >= 1) {
            propMap.put(REPLICA_TYPE, Replica.Type.NRT.name());
        } else if (numTlogReplicas >= 1) {
            propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name());
        } else {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " + Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0");
        }
        // Get the first node matching the shard to restore in
        String node;
        for (Map.Entry<ReplicaAssigner.Position, String> pvn : positionVsNodes.entrySet()) {
            ReplicaAssigner.Position position = pvn.getKey();
            if (position.shard == slice.getName()) {
                node = pvn.getValue();
                propMap.put(CoreAdminParams.NODE, node);
                positionVsNodes.remove(position);
                break;
            }
        }
        // add async param
        if (asyncId != null) {
            propMap.put(ASYNC, asyncId);
        }
        ocmh.addPropertyParams(message, propMap);
        ocmh.addReplica(clusterState, new ZkNodeProps(propMap), new NamedList(), null);
    }
    //refresh the location copy of collection state
    restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
    //Copy data from backed up index to each replica
    for (Slice slice : restoreCollection.getSlices()) {
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString());
        params.set(NAME, "snapshot." + slice.getName());
        params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString());
        params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
        ocmh.sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap);
    }
    ocmh.processResponses(new NamedList(), shardHandler, true, "Could not restore core", asyncId, requestMap);
    //Mark all shards in ACTIVE STATE
    {
        HashMap<String, Object> propMap = new HashMap<>();
        propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
        propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
        for (Slice shard : restoreCollection.getSlices()) {
            propMap.put(shard.getName(), Slice.State.ACTIVE.toString());
        }
        inQueue.offer(Utils.toJSON(new ZkNodeProps(propMap)));
    }
    //refresh the location copy of collection state
    restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
    if (totalReplicasPerShard > 1) {
        log.info("Adding replicas to restored collection={}", restoreCollection);
        for (Slice slice : restoreCollection.getSlices()) {
            //Add the remaining replicas for each shard, considering it's type
            int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0;
            // We already created either a NRT or an TLOG replica as leader
            if (numNrtReplicas > 0) {
                createdNrtReplicas++;
            } else if (createdTlogReplicas > 0) {
                createdTlogReplicas++;
            }
            for (int i = 1; i < totalReplicasPerShard; i++) {
                Replica.Type typeToCreate;
                if (createdNrtReplicas < numNrtReplicas) {
                    createdNrtReplicas++;
                    typeToCreate = Replica.Type.NRT;
                } else if (createdTlogReplicas < numTlogReplicas) {
                    createdTlogReplicas++;
                    typeToCreate = Replica.Type.TLOG;
                } else {
                    createdPullReplicas++;
                    typeToCreate = Replica.Type.PULL;
                    assert createdPullReplicas <= numPullReplicas : "Unexpected number of replicas";
                }
                log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate);
                HashMap<String, Object> propMap = new HashMap<>();
                propMap.put(COLLECTION_PROP, restoreCollectionName);
                propMap.put(SHARD_ID_PROP, slice.getName());
                propMap.put(REPLICA_TYPE, typeToCreate.name());
                // Get the first node matching the shard to restore in
                String node;
                for (Map.Entry<ReplicaAssigner.Position, String> pvn : positionVsNodes.entrySet()) {
                    ReplicaAssigner.Position position = pvn.getKey();
                    if (position.shard == slice.getName()) {
                        node = pvn.getValue();
                        propMap.put(CoreAdminParams.NODE, node);
                        positionVsNodes.remove(position);
                        break;
                    }
                }
                // add async param
                if (asyncId != null) {
                    propMap.put(ASYNC, asyncId);
                }
                ocmh.addPropertyParams(message, propMap);
                ocmh.addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null);
            }
        }
    }
    log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName);
}

Example 89 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class ZkCLI method main.

/**
   * Allows you to perform a variety of zookeeper related tasks, such as:
   * 
   * Bootstrap the current configs for all collections in solr.xml.
   * 
   * Upload a named config set from a given directory.
   * 
   * Link a named config set explicity to a collection.
   * 
   * Clear ZooKeeper info.
   * 
   * If you also pass a solrPort, it will be used to start an embedded zk useful
   * for single machine, multi node tests.
   */
public static void main(String[] args) throws InterruptedException, TimeoutException, IOException, ParserConfigurationException, SAXException, KeeperException {
    CommandLineParser parser = new PosixParser();
    Options options = new Options();
    options.addOption(OptionBuilder.hasArg(true).withDescription("cmd to run: " + BOOTSTRAP + ", " + UPCONFIG + ", " + DOWNCONFIG + ", " + LINKCONFIG + ", " + MAKEPATH + ", " + PUT + ", " + PUT_FILE + "," + GET + "," + GET_FILE + ", " + LIST + ", " + CLEAR + ", " + UPDATEACLS + ", " + LS).create(CMD));
    Option zkHostOption = new Option("z", ZKHOST, true, "ZooKeeper host address");
    options.addOption(zkHostOption);
    Option solrHomeOption = new Option("s", SOLRHOME, true, "for " + BOOTSTRAP + ", " + RUNZK + ": solrhome location");
    options.addOption(solrHomeOption);
    options.addOption("d", CONFDIR, true, "for " + UPCONFIG + ": a directory of configuration files");
    options.addOption("n", CONFNAME, true, "for " + UPCONFIG + ", " + LINKCONFIG + ": name of the config set");
    options.addOption("c", COLLECTION, true, "for " + LINKCONFIG + ": name of the collection");
    options.addOption(EXCLUDE_REGEX_SHORT, EXCLUDE_REGEX, true, "for " + UPCONFIG + ": files matching this regular expression won't be uploaded");
    options.addOption("r", RUNZK, true, "run zk internally by passing the solr run port - only for clusters on one machine (tests, dev)");
    options.addOption("h", HELP, false, "bring up this help page");
    options.addOption(NAME, true, "name of the cluster property to set");
    options.addOption(VALUE_LONG, true, "value of the cluster to set");
    try {
        // parse the command line arguments
        CommandLine line = parser.parse(options, args);
        if (line.hasOption(HELP) || !line.hasOption(ZKHOST) || !line.hasOption(CMD)) {
            // automatically generate the help statement
            HelpFormatter formatter = new HelpFormatter();
            formatter.printHelp(ZK_CLI_NAME, options);
            stdout.println("Examples:");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + BOOTSTRAP + " -" + SOLRHOME + " /opt/solr");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + UPCONFIG + " -" + CONFDIR + " /opt/solr/collection1/conf" + " -" + CONFNAME + " myconf");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + DOWNCONFIG + " -" + CONFDIR + " /opt/solr/collection1/conf" + " -" + CONFNAME + " myconf");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT_FILE + " /solr.xml /User/myuser/solr/solr.xml");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET + " /solr.xml");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET_FILE + " /solr.xml solr.xml.file");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + LS + " /solr/live_nodes");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLUSTERPROP + " -" + NAME + " urlScheme -" + VALUE_LONG + " https");
            stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + UPDATEACLS + " /solr");
            return;
        }
        // start up a tmp zk server first
        String zkServerAddress = line.getOptionValue(ZKHOST);
        String solrHome = line.getOptionValue(SOLRHOME);
        String solrPort = null;
        if (line.hasOption(RUNZK)) {
            if (!line.hasOption(SOLRHOME)) {
                stdout.println("-" + SOLRHOME + " is required for " + RUNZK);
                System.exit(1);
            }
            solrPort = line.getOptionValue(RUNZK);
        }
        SolrZkServer zkServer = null;
        if (solrPort != null) {
            zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data", solrHome, Integer.parseInt(solrPort));
            zkServer.parseConfig();
            zkServer.start();
        }
        SolrZkClient zkClient = null;
        try {
            zkClient = new SolrZkClient(zkServerAddress, 30000, 30000, () -> {
            });
            if (line.getOptionValue(CMD).equalsIgnoreCase(BOOTSTRAP)) {
                if (!line.hasOption(SOLRHOME)) {
                    stdout.println("-" + SOLRHOME + " is required for " + BOOTSTRAP);
                    System.exit(1);
                }
                CoreContainer cc = new CoreContainer(solrHome);
                if (!ZkController.checkChrootPath(zkServerAddress, true)) {
                    stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
                    System.exit(1);
                }
                ZkController.bootstrapConf(zkClient, cc, solrHome);
            // No need to close the CoreContainer, as it wasn't started
            // up in the first place...
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(UPCONFIG)) {
                if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
                    stdout.println("-" + CONFDIR + " and -" + CONFNAME + " are required for " + UPCONFIG);
                    System.exit(1);
                }
                String confDir = line.getOptionValue(CONFDIR);
                String confName = line.getOptionValue(CONFNAME);
                final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT);
                if (!ZkController.checkChrootPath(zkServerAddress, true)) {
                    stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
                    System.exit(1);
                }
                ZkConfigManager configManager = new ZkConfigManager(zkClient);
                final Pattern excludePattern = Pattern.compile(excludeExpr);
                configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern);
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) {
                if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
                    stdout.println("-" + CONFDIR + " and -" + CONFNAME + " are required for " + DOWNCONFIG);
                    System.exit(1);
                }
                String confDir = line.getOptionValue(CONFDIR);
                String confName = line.getOptionValue(CONFNAME);
                ZkConfigManager configManager = new ZkConfigManager(zkClient);
                configManager.downloadConfigDir(confName, Paths.get(confDir));
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(LINKCONFIG)) {
                if (!line.hasOption(COLLECTION) || !line.hasOption(CONFNAME)) {
                    stdout.println("-" + COLLECTION + " and -" + CONFNAME + " are required for " + LINKCONFIG);
                    System.exit(1);
                }
                String collection = line.getOptionValue(COLLECTION);
                String confName = line.getOptionValue(CONFNAME);
                ZkController.linkConfSet(zkClient, collection, confName);
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(LIST)) {
                zkClient.printLayoutToStream(stdout);
            } else if (line.getOptionValue(CMD).equals(LS)) {
                List argList = line.getArgList();
                if (argList.size() != 1) {
                    stdout.println("-" + LS + " requires one arg - the path to list");
                    System.exit(1);
                }
                StringBuilder sb = new StringBuilder();
                String path = argList.get(0).toString();
                zkClient.printLayout(path == null ? "/" : path, 0, sb);
                stdout.println(sb.toString());
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(CLEAR)) {
                List arglist = line.getArgList();
                if (arglist.size() != 1) {
                    stdout.println("-" + CLEAR + " requires one arg - the path to clear");
                    System.exit(1);
                }
                zkClient.clean(arglist.get(0).toString());
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(MAKEPATH)) {
                List arglist = line.getArgList();
                if (arglist.size() != 1) {
                    stdout.println("-" + MAKEPATH + " requires one arg - the path to make");
                    System.exit(1);
                }
                zkClient.makePath(arglist.get(0).toString(), true);
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(PUT)) {
                List arglist = line.getArgList();
                if (arglist.size() != 2) {
                    stdout.println("-" + PUT + " requires two args - the path to create and the data string");
                    System.exit(1);
                }
                String path = arglist.get(0).toString();
                if (zkClient.exists(path, true)) {
                    zkClient.setData(path, arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), true);
                } else {
                    zkClient.create(path, arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, true);
                }
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(PUT_FILE)) {
                List arglist = line.getArgList();
                if (arglist.size() != 2) {
                    stdout.println("-" + PUT_FILE + " requires two args - the path to create in ZK and the path to the local file");
                    System.exit(1);
                }
                String path = arglist.get(0).toString();
                InputStream is = new FileInputStream(arglist.get(1).toString());
                try {
                    if (zkClient.exists(path, true)) {
                        zkClient.setData(path, IOUtils.toByteArray(is), true);
                    } else {
                        zkClient.create(path, IOUtils.toByteArray(is), CreateMode.PERSISTENT, true);
                    }
                } finally {
                    IOUtils.closeQuietly(is);
                }
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(GET)) {
                List arglist = line.getArgList();
                if (arglist.size() != 1) {
                    stdout.println("-" + GET + " requires one arg - the path to get");
                    System.exit(1);
                }
                byte[] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
                stdout.println(new String(data, StandardCharsets.UTF_8));
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(GET_FILE)) {
                List arglist = line.getArgList();
                if (arglist.size() != 2) {
                    stdout.println("-" + GET_FILE + "requires two args - the path to get and the file to save it to");
                    System.exit(1);
                }
                byte[] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
                FileUtils.writeByteArrayToFile(new File(arglist.get(1).toString()), data);
            } else if (line.getOptionValue(CMD).equals(UPDATEACLS)) {
                List arglist = line.getArgList();
                if (arglist.size() != 1) {
                    stdout.println("-" + UPDATEACLS + " requires one arg - the path to update");
                    System.exit(1);
                }
                zkClient.updateACLs(arglist.get(0).toString());
            } else if (line.getOptionValue(CMD).equalsIgnoreCase(CLUSTERPROP)) {
                if (!line.hasOption(NAME)) {
                    stdout.println("-" + NAME + " is required for " + CLUSTERPROP);
                }
                String propertyName = line.getOptionValue(NAME);
                //If -val option is missing, we will use the null value. This is required to maintain
                //compatibility with Collections API.
                String propertyValue = line.getOptionValue(VALUE_LONG);
                ClusterProperties props = new ClusterProperties(zkClient);
                try {
                    props.setClusterProperty(propertyName, propertyValue);
                } catch (IOException ex) {
                    stdout.println("Unable to set the cluster property due to following error : " + ex.getLocalizedMessage());
                    System.exit(1);
                }
            } else {
                // If not cmd matches
                stdout.println("Unknown command " + line.getOptionValue(CMD) + ". Use -h to get help.");
                System.exit(1);
            }
        } finally {
            if (solrPort != null) {
                zkServer.stop();
            }
            if (zkClient != null) {
                zkClient.close();
            }
        }
    } catch (ParseException exp) {
        stdout.println("Unexpected exception:" + exp.getMessage());
    }
}

Example 90 with CoreContainer

use of org.apache.solr.core.CoreContainer in project lucene-solr by apache.

the class ScoreJoinQParserPlugin method createParser.

@Override
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
    return new QParser(qstr, localParams, params, req) {

        @Override
        public Query parse() throws SyntaxError {
            final String fromField = localParams.get("from");
            final String fromIndex = localParams.get("fromIndex");
            final String toField = localParams.get("to");
            final ScoreMode scoreMode = ScoreModeParser.parse(getParam(SCORE));
            final String v = localParams.get(CommonParams.VALUE);
            final Query q = createQuery(fromField, v, fromIndex, toField, scoreMode, CommonParams.TRUE.equals(localParams.get("TESTenforceSameCoreAsAnotherOne")));
            return q;
        }

        private Query createQuery(final String fromField, final String fromQueryStr, String fromIndex, final String toField, final ScoreMode scoreMode, boolean byPassShortCircutCheck) throws SyntaxError {
            final String myCore = req.getCore().getCoreDescriptor().getName();
            if (fromIndex != null && (!fromIndex.equals(myCore) || byPassShortCircutCheck)) {
                CoreContainer container = req.getCore().getCoreContainer();
                final String coreName = getCoreName(fromIndex, container);
                final SolrCore fromCore = container.getCore(coreName);
                RefCounted<SolrIndexSearcher> fromHolder = null;
                if (fromCore == null) {
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cross-core join: no such core " + coreName);
                }
                long fromCoreOpenTime = 0;
                LocalSolrQueryRequest otherReq = new LocalSolrQueryRequest(fromCore, params);
                try {
                    QParser fromQueryParser = QParser.getParser(fromQueryStr, otherReq);
                    Query fromQuery = fromQueryParser.getQuery();
                    fromHolder = fromCore.getRegisteredSearcher();
                    if (fromHolder != null) {
                        fromCoreOpenTime = fromHolder.get().getOpenNanoTime();
                    }
                    return new OtherCoreJoinQuery(fromQuery, fromField, coreName, fromCoreOpenTime, scoreMode, toField);
                } finally {
                    otherReq.close();
                    fromCore.close();
                    if (fromHolder != null)
                        fromHolder.decref();
                }
            } else {
                QParser fromQueryParser = subQuery(fromQueryStr, null);
                final Query fromQuery = fromQueryParser.getQuery();
                return new SameCoreJoinQuery(fromQuery, fromField, toField, scoreMode);
            }
        }
    };
}