Examples with ZkController org.apache.solr.cloud.ZkController used on opensource projects

Example 6 with ZkController

use of org.apache.solr.cloud.ZkController in project lucene-solr by apache.

the class HttpShardHandler method prepDistributed.

@Override
public void prepDistributed(ResponseBuilder rb) {
    final SolrQueryRequest req = rb.req;
    final SolrParams params = req.getParams();
    final String shards = params.get(ShardParams.SHARDS);
    // since the cost of grabbing cloud state is still up in the air, we grab it only
    // if we need it.
    ClusterState clusterState = null;
    Map<String, Slice> slices = null;
    CoreDescriptor coreDescriptor = req.getCore().getCoreDescriptor();
    CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
    ZkController zkController = req.getCore().getCoreContainer().getZkController();
    final ReplicaListTransformer replicaListTransformer = httpShardHandlerFactory.getReplicaListTransformer(req);
    if (shards != null) {
        List<String> lst = StrUtils.splitSmart(shards, ",", true);
        rb.shards = lst.toArray(new String[lst.size()]);
        rb.slices = new String[rb.shards.length];
        if (zkController != null) {
            // figure out which shards are slices
            for (int i = 0; i < rb.shards.length; i++) {
                if (rb.shards[i].indexOf('/') < 0) {
                    // this is a logical shard
                    rb.slices[i] = rb.shards[i];
                    rb.shards[i] = null;
                }
            }
        }
    } else if (zkController != null) {
        // we weren't provided with an explicit list of slices to query via "shards", so use the cluster state
        clusterState = zkController.getClusterState();
        String shardKeys = params.get(ShardParams._ROUTE_);
        // This will be the complete list of slices we need to query for this request.
        slices = new HashMap<>();
        // we need to find out what collections this request is for.
        // A comma-separated list of specified collections.
        // Eg: "collection1,collection2,collection3"
        String collections = params.get("collection");
        if (collections != null) {
            // If there were one or more collections specified in the query, split
            // each parameter and store as a separate member of a List.
            List<String> collectionList = StrUtils.splitSmart(collections, ",", true);
            // cloud state and add them to the Map 'slices'.
            for (String collectionName : collectionList) {
                // The original code produced <collection-name>_<shard-name> when the collections
                // parameter was specified (see ClientUtils.appendMap)
                // Is this necessary if ony one collection is specified?
                // i.e. should we change multiCollection to collectionList.size() > 1?
                addSlices(slices, clusterState, params, collectionName, shardKeys, true);
            }
        } else {
            // just this collection
            String collectionName = cloudDescriptor.getCollectionName();
            addSlices(slices, clusterState, params, collectionName, shardKeys, false);
        }
        // Store the logical slices in the ResponseBuilder and create a new
        // String array to hold the physical shards (which will be mapped
        // later).
        rb.slices = slices.keySet().toArray(new String[slices.size()]);
        rb.shards = new String[rb.slices.length];
    }
    //
    if (zkController != null) {
        // Are we hosting the shard that this request is for, and are we active? If so, then handle it ourselves
        // and make it a non-distributed request.
        String ourSlice = cloudDescriptor.getShardId();
        String ourCollection = cloudDescriptor.getCollectionName();
        // Some requests may only be fulfilled by replicas of type Replica.Type.NRT
        boolean onlyNrtReplicas = Boolean.TRUE == req.getContext().get(ONLY_NRT_REPLICAS);
        if (rb.slices.length == 1 && rb.slices[0] != null && // handle the <collection>_<slice> format
        (rb.slices[0].equals(ourSlice) || rb.slices[0].equals(ourCollection + "_" + ourSlice)) && cloudDescriptor.getLastPublished() == Replica.State.ACTIVE && (!onlyNrtReplicas || cloudDescriptor.getReplicaType() == Replica.Type.NRT)) {
            // currently just a debugging parameter to check distrib search on a single node
            boolean shortCircuit = params.getBool("shortCircuit", true);
            String targetHandler = params.get(ShardParams.SHARDS_QT);
            // if a different handler is specified, don't short-circuit
            shortCircuit = shortCircuit && targetHandler == null;
            if (shortCircuit) {
                rb.isDistrib = false;
                rb.shortCircuitedURL = ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), coreDescriptor.getName());
                return;
            }
        // We shouldn't need to do anything to handle "shard.rows" since it was previously meant to be an optimization?
        }
        for (int i = 0; i < rb.shards.length; i++) {
            final List<String> shardUrls;
            if (rb.shards[i] != null) {
                shardUrls = StrUtils.splitSmart(rb.shards[i], "|", true);
                replicaListTransformer.transform(shardUrls);
            } else {
                if (clusterState == null) {
                    clusterState = zkController.getClusterState();
                    slices = clusterState.getSlicesMap(cloudDescriptor.getCollectionName());
                }
                String sliceName = rb.slices[i];
                Slice slice = slices.get(sliceName);
                if (slice == null) {
                    // Treat this the same as "all servers down" for a slice, and let things continue
                    // if partial results are acceptable
                    rb.shards[i] = "";
                    continue;
                // throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such shard: " + sliceName);
                }
                final Predicate<Replica> isShardLeader = new Predicate<Replica>() {

                    private Replica shardLeader = null;

                    @Override
                    public boolean test(Replica replica) {
                        if (shardLeader == null) {
                            try {
                                shardLeader = zkController.getZkStateReader().getLeaderRetry(cloudDescriptor.getCollectionName(), slice.getName());
                            } catch (InterruptedException e) {
                                throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + slice.getName() + " in collection " + cloudDescriptor.getCollectionName(), e);
                            } catch (SolrException e) {
                                if (log.isDebugEnabled()) {
                                    log.debug("Exception finding leader for shard {} in collection {}. Collection State: {}", slice.getName(), cloudDescriptor.getCollectionName(), zkController.getZkStateReader().getClusterState().getCollectionOrNull(cloudDescriptor.getCollectionName()));
                                }
                                throw e;
                            }
                        }
                        return replica.getName().equals(shardLeader.getName());
                    }
                };
                final List<Replica> eligibleSliceReplicas = collectEligibleReplicas(slice, clusterState, onlyNrtReplicas, isShardLeader);
                replicaListTransformer.transform(eligibleSliceReplicas);
                shardUrls = new ArrayList<>(eligibleSliceReplicas.size());
                for (Replica replica : eligibleSliceReplicas) {
                    String url = ZkCoreNodeProps.getCoreUrl(replica);
                    shardUrls.add(url);
                }
                if (shardUrls.isEmpty()) {
                    boolean tolerant = rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false);
                    if (!tolerant) {
                        // stop the check when there are no replicas available for a shard
                        throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "no servers hosting shard: " + rb.slices[i]);
                    }
                }
            }
            // And now recreate the | delimited list of equivalent servers
            rb.shards[i] = createSliceShardsStr(shardUrls);
        }
    }
    String shards_rows = params.get(ShardParams.SHARDS_ROWS);
    if (shards_rows != null) {
        rb.shards_rows = Integer.parseInt(shards_rows);
    }
    String shards_start = params.get(ShardParams.SHARDS_START);
    if (shards_start != null) {
        rb.shards_start = Integer.parseInt(shards_start);
    }
}

Example 7 with ZkController

use of org.apache.solr.cloud.ZkController in project lucene-solr by apache.

the class ZookeeperInfoHandler method handleRequestBody.

@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    final SolrParams params = req.getParams();
    Map<String, String> map = new HashMap<>(1);
    map.put(WT, "raw");
    map.put(OMIT_HEADER, "true");
    req.setParams(SolrParams.wrapDefaults(new MapSolrParams(map), params));
    synchronized (this) {
        if (pagingSupport == null) {
            pagingSupport = new PagedCollectionSupport();
            ZkController zkController = cores.getZkController();
            if (zkController != null) {
                // get notified when the ZK session expires (so we can clear the cached collections and rebuild)
                zkController.addOnReconnectListener(pagingSupport);
            }
        }
    }
    String path = params.get(PATH);
    String addr = params.get("addr");
    if (addr != null && addr.length() == 0) {
        addr = null;
    }
    String detailS = params.get("detail");
    boolean detail = detailS != null && detailS.equals("true");
    String dumpS = params.get("dump");
    boolean dump = dumpS != null && dumpS.equals("true");
    int start = params.getInt("start", 0);
    int rows = params.getInt("rows", -1);
    String filterType = params.get("filterType");
    if (filterType != null) {
        filterType = filterType.trim().toLowerCase(Locale.ROOT);
        if (filterType.length() == 0)
            filterType = null;
    }
    FilterType type = (filterType != null) ? FilterType.valueOf(filterType) : FilterType.none;
    String filter = (type != FilterType.none) ? params.get("filter") : null;
    if (filter != null) {
        filter = filter.trim();
        if (filter.length() == 0)
            filter = null;
    }
    ZKPrinter printer = new ZKPrinter(cores.getZkController(), addr);
    printer.detail = detail;
    printer.dump = dump;
    boolean isGraphView = "graph".equals(params.get("view"));
    printer.page = (isGraphView && "/clusterstate.json".equals(path)) ? new PageOfCollections(start, rows, type, filter) : null;
    printer.pagingSupport = pagingSupport;
    try {
        printer.print(path);
    } finally {
        printer.close();
    }
    rsp.getValues().add(RawResponseWriter.CONTENT, printer);
}

Example 8 with ZkController

use of org.apache.solr.cloud.ZkController in project lucene-solr by apache.

the class RequestSyncShardOp method execute.

@Override
public void execute(CallInfo it) throws Exception {
    final SolrParams params = it.req.getParams();
    log.info("I have been requested to sync up my shard");
    ZkController zkController = it.handler.coreContainer.getZkController();
    if (zkController == null) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only valid for SolrCloud");
    }
    String cname = params.get(CoreAdminParams.CORE);
    if (cname == null) {
        throw new IllegalArgumentException(CoreAdminParams.CORE + " is required");
    }
    SyncStrategy syncStrategy = null;
    try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
        if (core != null) {
            syncStrategy = new SyncStrategy(core.getCoreContainer());
            Map<String, Object> props = new HashMap<>();
            props.put(ZkStateReader.BASE_URL_PROP, zkController.getBaseUrl());
            props.put(ZkStateReader.CORE_NAME_PROP, cname);
            props.put(ZkStateReader.NODE_NAME_PROP, zkController.getNodeName());
            boolean success = syncStrategy.sync(zkController, core, new ZkNodeProps(props), true).isSuccess();
            // solrcloud_debug
            if (log.isDebugEnabled()) {
                try {
                    RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
                    SolrIndexSearcher searcher = searchHolder.get();
                    try {
                        log.debug(core.getCoreContainer().getZkController().getNodeName() + " synched " + searcher.search(new MatchAllDocsQuery(), 1).totalHits);
                    } finally {
                        searchHolder.decref();
                    }
                } catch (Exception e) {
                    log.debug("Error in solrcloud_debug block", e);
                }
            }
            if (!success) {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Sync Failed");
            }
        } else {
            SolrException.log(log, "Could not find core to call sync:" + cname);
        }
    } finally {
        // no recoveryStrat close for now
        if (syncStrategy != null) {
            syncStrategy.close();
        }
    }
}

Example 9 with ZkController

use of org.apache.solr.cloud.ZkController in project lucene-solr by apache.

the class ZkContainer method initZooKeeper.

public void initZooKeeper(final CoreContainer cc, String solrHome, CloudConfig config) {
    ZkController zkController = null;
    String zkRun = System.getProperty("zkRun");
    if (zkRun != null && config == null)
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Cannot start Solr in cloud mode - no cloud config provided");
    if (config == null)
        // not in zk mode
        return;
    String zookeeperHost = config.getZkHost();
    // zookeeper in quorum mode currently causes a failure when trying to
    // register log4j mbeans.  See SOLR-2369
    // TODO: remove after updating to an slf4j based zookeeper
    System.setProperty("zookeeper.jmx.log4j.disable", "true");
    if (zkRun != null) {
        String zkDataHome = System.getProperty("zkServerDataDir", Paths.get(solrHome).resolve("zoo_data").toString());
        String zkConfHome = System.getProperty("zkServerConfDir", solrHome);
        zkServer = new SolrZkServer(stripChroot(zkRun), stripChroot(config.getZkHost()), zkDataHome, zkConfHome, config.getSolrHostPort());
        zkServer.parseConfig();
        zkServer.start();
        // set client from server config if not already set
        if (zookeeperHost == null) {
            zookeeperHost = zkServer.getClientString();
        }
    }
    int zkClientConnectTimeout = 30000;
    if (zookeeperHost != null) {
        // we are ZooKeeper enabled
        try {
            // If this is an ensemble, allow for a long connect time for other servers to come up
            if (zkRun != null && zkServer.getServers().size() > 1) {
                // 1 day for embedded ensemble
                zkClientConnectTimeout = 24 * 60 * 60 * 1000;
                log.info("Zookeeper client=" + zookeeperHost + "  Waiting for a quorum.");
            } else {
                log.info("Zookeeper client=" + zookeeperHost);
            }
            String confDir = System.getProperty("bootstrap_confdir");
            boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
            if (!ZkController.checkChrootPath(zookeeperHost, (confDir != null) || boostrapConf || zkRunOnly)) {
                throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A chroot was specified in ZkHost but the znode doesn't exist. " + zookeeperHost);
            }
            zkController = new ZkController(cc, zookeeperHost, zkClientConnectTimeout, config, new CurrentCoreDescriptorProvider() {

                @Override
                public List<CoreDescriptor> getCurrentDescriptors() {
                    List<CoreDescriptor> descriptors = new ArrayList<>(cc.getLoadedCoreNames().size());
                    Collection<SolrCore> cores = cc.getCores();
                    for (SolrCore core : cores) {
                        descriptors.add(core.getCoreDescriptor());
                    }
                    return descriptors;
                }
            });
            if (zkRun != null && zkServer.getServers().size() > 1 && confDir == null && boostrapConf == false) {
                // we are part of an ensemble and we are not uploading the config - pause to give the config time
                // to get up
                Thread.sleep(10000);
            }
            if (confDir != null) {
                Path configPath = Paths.get(confDir);
                if (!Files.isDirectory(configPath))
                    throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files");
                String confName = System.getProperty(ZkController.COLLECTION_PARAM_PREFIX + ZkController.CONFIGNAME_PROP, "configuration1");
                ZkConfigManager configManager = new ZkConfigManager(zkController.getZkClient());
                configManager.uploadConfigDir(configPath, confName);
            }
            if (boostrapConf) {
                ZkController.bootstrapConf(zkController.getZkClient(), cc, solrHome);
            }
        } catch (InterruptedException e) {
            // Restore the interrupted status
            Thread.currentThread().interrupt();
            log.error("", e);
            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
        } catch (TimeoutException e) {
            log.error("Could not connect to ZooKeeper", e);
            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
        } catch (IOException | KeeperException e) {
            log.error("", e);
            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
        }
    }
    this.zkController = zkController;
}

Example 10 with ZkController

use of org.apache.solr.cloud.ZkController in project lucene-solr by apache.

the class QueryElevationComponent method inform.

@Override
public void inform(SolrCore core) {
    IndexSchema schema = core.getLatestSchema();
    String a = initArgs.get(FIELD_TYPE);
    if (a != null) {
        FieldType ft = schema.getFieldTypes().get(a);
        if (ft == null) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown FieldType: '" + a + "' used in QueryElevationComponent");
        }
        analyzer = ft.getQueryAnalyzer();
    }
    SchemaField sf = schema.getUniqueKeyField();
    if (sf == null) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "QueryElevationComponent requires the schema to have a uniqueKeyField.");
    }
    idSchemaFT = sf.getType();
    idField = sf.getName();
    //register the EditorialMarkerFactory
    String excludeName = initArgs.get(QueryElevationParams.EXCLUDE_MARKER_FIELD_NAME, "excluded");
    if (excludeName == null || excludeName.equals("") == true) {
        excludeName = "excluded";
    }
    ExcludedMarkerFactory excludedMarkerFactory = new ExcludedMarkerFactory();
    core.addTransformerFactory(excludeName, excludedMarkerFactory);
    ElevatedMarkerFactory elevatedMarkerFactory = new ElevatedMarkerFactory();
    String markerName = initArgs.get(QueryElevationParams.EDITORIAL_MARKER_FIELD_NAME, "elevated");
    if (markerName == null || markerName.equals("") == true) {
        markerName = "elevated";
    }
    core.addTransformerFactory(markerName, elevatedMarkerFactory);
    forceElevation = initArgs.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation);
    try {
        synchronized (elevationCache) {
            elevationCache.clear();
            String f = initArgs.get(CONFIG_FILE);
            if (f == null) {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "QueryElevationComponent must specify argument: '" + CONFIG_FILE + "' -- path to elevate.xml");
            }
            boolean exists = false;
            // check if using ZooKeeper
            ZkController zkController = core.getCoreContainer().getZkController();
            if (zkController != null) {
                // TODO : shouldn't have to keep reading the config name when it has been read before
                exists = zkController.configFileExists(zkController.getZkStateReader().readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f);
            } else {
                File fC = new File(core.getResourceLoader().getConfigDir(), f);
                File fD = new File(core.getDataDir(), f);
                if (fC.exists() == fD.exists()) {
                    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "QueryElevationComponent missing config file: '" + f + "\n" + "either: " + fC.getAbsolutePath() + " or " + fD.getAbsolutePath() + " must exist, but not both.");
                }
                if (fC.exists()) {
                    exists = true;
                    log.info("Loading QueryElevation from: " + fC.getAbsolutePath());
                    Config cfg = new Config(core.getResourceLoader(), f);
                    elevationCache.put(null, loadElevationMap(cfg));
                }
            }
            //in other words, we think this is in the data dir, not the conf dir
            if (!exists) {
                // preload the first data
                RefCounted<SolrIndexSearcher> searchHolder = null;
                try {
                    searchHolder = core.getNewestSearcher(false);
                    IndexReader reader = searchHolder.get().getIndexReader();
                    getElevationMap(reader, core);
                } finally {
                    if (searchHolder != null)
                        searchHolder.decref();
                }
            }
        }
    } catch (Exception ex) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error initializing QueryElevationComponent.", ex);
    }
}