Examples with CloudSolrClient org.apache.solr.client.solrj.impl.CloudSolrClient used on opensource projects

Example 71 with CloudSolrClient

use of org.apache.solr.client.solrj.impl.CloudSolrClient in project metron by apache.

the class SolrComponent method getAllIndexedDocs.

public List<Map<String, Object>> getAllIndexedDocs(String collection) {
    List<Map<String, Object>> docs = new ArrayList<>();
    CloudSolrClient solr = miniSolrCloudCluster.getSolrClient();
    solr.setDefaultCollection(collection);
    SolrQuery parameters = new SolrQuery();
    parameters.set("q", "*:*");
    try {
        solr.commit();
        QueryResponse response = solr.query(parameters);
        for (SolrDocument solrDocument : response.getResults()) {
            docs.add(solrDocument);
        }
    } catch (SolrServerException | IOException e) {
        e.printStackTrace();
    }
    return docs;
}

Example 72 with CloudSolrClient

use of org.apache.solr.client.solrj.impl.CloudSolrClient in project nutch by apache.

the class SolrUtils method getCloudSolrClient.

public static CloudSolrClient getCloudSolrClient(String url) throws MalformedURLException {
    CloudSolrClient sc = new CloudSolrClient(url.replace('|', ','));
    sc.setParallelUpdates(true);
    sc.connect();
    return sc;
}

Example 73 with CloudSolrClient

use of org.apache.solr.client.solrj.impl.CloudSolrClient in project storm by apache.

the class SolrUpdateBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    this.solrClient = new CloudSolrClient(solrConfig.getZkHostString());
    this.toCommitTuples = new ArrayList<>(capacity());
}

Example 74 with CloudSolrClient

use of org.apache.solr.client.solrj.impl.CloudSolrClient in project gora by apache.

the class SolrStore method initialize.

/**
   * Initialize the data store by reading the credentials, setting the client's properties up and
   * reading the mapping file. Initialize is called when then the call to
   * {@link org.apache.gora.store.DataStoreFactory#createDataStore} is made.
   *
   * @param keyClass
   * @param persistentClass
   * @param properties
   */
@Override
public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) {
    super.initialize(keyClass, persistentClass, properties);
    try {
        String mappingFile = DataStoreFactory.getMappingFile(properties, this, DEFAULT_MAPPING_FILE);
        mapping = readMapping(mappingFile);
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
    }
    SolrClientUrl = DataStoreFactory.findProperty(properties, this, SOLR_URL_PROPERTY, null);
    solrConfig = DataStoreFactory.findProperty(properties, this, SOLR_CONFIG_PROPERTY, null);
    solrSchema = DataStoreFactory.findProperty(properties, this, SOLR_SCHEMA_PROPERTY, null);
    solrJServerImpl = DataStoreFactory.findProperty(properties, this, SOLR_SOLRJSERVER_IMPL, "http");
    serverUserAuth = DataStoreFactory.findBooleanProperty(properties, this, SOLR_SERVER_USER_AUTH, "false");
    if (serverUserAuth) {
        serverUsername = DataStoreFactory.findProperty(properties, this, SOLR_SERVER_USERNAME, null);
        serverPassword = DataStoreFactory.findProperty(properties, this, SOLR_SERVER_PASSWORD, null);
    }
    LOG.info("Using Solr server at " + SolrClientUrl);
    String solrJServerType = ((solrJServerImpl == null || solrJServerImpl.equals("")) ? "http" : solrJServerImpl);
    // HttpSolrClient - denoted by "http" in properties
    if (solrJServerType.toLowerCase(Locale.getDefault()).equals("http")) {
        LOG.info("Using HttpSolrClient Solrj implementation.");
        this.adminServer = new HttpSolrClient(SolrClientUrl);
        this.server = new HttpSolrClient(SolrClientUrl + "/" + mapping.getCoreName());
        if (serverUserAuth) {
            HttpClientUtil.setBasicAuth((DefaultHttpClient) ((HttpSolrClient) adminServer).getHttpClient(), serverUsername, serverPassword);
            HttpClientUtil.setBasicAuth((DefaultHttpClient) ((HttpSolrClient) server).getHttpClient(), serverUsername, serverPassword);
        }
    // CloudSolrClient - denoted by "cloud" in properties
    } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("cloud")) {
        LOG.info("Using CloudSolrClient Solrj implementation.");
        this.adminServer = new CloudSolrClient(SolrClientUrl);
        this.server = new CloudSolrClient(SolrClientUrl + "/" + mapping.getCoreName());
        if (serverUserAuth) {
            HttpClientUtil.setBasicAuth((DefaultHttpClient) ((CloudSolrClient) adminServer).getLbClient().getHttpClient(), serverUsername, serverPassword);
            HttpClientUtil.setBasicAuth((DefaultHttpClient) ((CloudSolrClient) server).getLbClient().getHttpClient(), serverUsername, serverPassword);
        }
    } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("concurrent")) {
        LOG.info("Using ConcurrentUpdateSolrClient Solrj implementation.");
        this.adminServer = new ConcurrentUpdateSolrClient(SolrClientUrl, 1000, 10);
        this.server = new ConcurrentUpdateSolrClient(SolrClientUrl + "/" + mapping.getCoreName(), 1000, 10);
    // LBHttpSolrClient - denoted by "loadbalance" in properties
    } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("loadbalance")) {
        LOG.info("Using LBHttpSolrClient Solrj implementation.");
        String[] solrUrlElements = StringUtils.split(SolrClientUrl);
        try {
            this.adminServer = new LBHttpSolrClient(solrUrlElements);
        } catch (MalformedURLException e) {
            LOG.error(e.getMessage());
            throw new RuntimeException(e);
        }
        try {
            this.server = new LBHttpSolrClient(solrUrlElements + "/" + mapping.getCoreName());
        } catch (MalformedURLException e) {
            LOG.error(e.getMessage());
            throw new RuntimeException(e);
        }
        if (serverUserAuth) {
            HttpClientUtil.setBasicAuth((DefaultHttpClient) ((LBHttpSolrClient) adminServer).getHttpClient(), serverUsername, serverPassword);
            HttpClientUtil.setBasicAuth((DefaultHttpClient) ((LBHttpSolrClient) server).getHttpClient(), serverUsername, serverPassword);
        }
    }
    if (autoCreateSchema) {
        createSchema();
    }
    String batchSizeString = DataStoreFactory.findProperty(properties, this, SOLR_BATCH_SIZE_PROPERTY, null);
    if (batchSizeString != null) {
        try {
            batchSize = Integer.parseInt(batchSizeString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid batch size '{}', using default {}", batchSizeString, DEFAULT_BATCH_SIZE);
        }
    }
    batch = new ArrayList<>(batchSize);
    String commitWithinString = DataStoreFactory.findProperty(properties, this, SOLR_COMMIT_WITHIN_PROPERTY, null);
    if (commitWithinString != null) {
        try {
            commitWithin = Integer.parseInt(commitWithinString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid commit within '{}' , using default {}", commitWithinString, DEFAULT_COMMIT_WITHIN);
        }
    }
    String resultsSizeString = DataStoreFactory.findProperty(properties, this, SOLR_RESULTS_SIZE_PROPERTY, null);
    if (resultsSizeString != null) {
        try {
            resultsSize = Integer.parseInt(resultsSizeString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid results size '{}' , using default {}", resultsSizeString, DEFAULT_RESULTS_SIZE);
        }
    }
}

Example 75 with CloudSolrClient

use of org.apache.solr.client.solrj.impl.CloudSolrClient in project lucene-solr by apache.

the class TestTlogReplica method testRecovery.

@SuppressWarnings("unchecked")
public void testRecovery() throws Exception {
    boolean useKill = random().nextBoolean();
    createAndWaitForCollection(1, 0, 2, 0);
    CloudSolrClient cloudClient = cluster.getSolrClient();
    new UpdateRequest().add(sdoc("id", "3")).add(sdoc("id", "4")).commit(cloudClient, collectionName);
    new UpdateRequest().add(sdoc("id", "5")).process(cloudClient, collectionName);
    JettySolrRunner solrRunner = getSolrRunner(false).get(0);
    if (useKill) {
        ChaosMonkey.kill(solrRunner);
    } else {
        ChaosMonkey.stop(solrRunner);
    }
    waitForState("Replica still up", collectionName, activeReplicaCount(0, 1, 0));
    new UpdateRequest().add(sdoc("id", "6")).process(cloudClient, collectionName);
    ChaosMonkey.start(solrRunner);
    waitForState("Replica didn't recover", collectionName, activeReplicaCount(0, 2, 0));
    // We skip peerSync, so replica will always trigger commit on leader
    // We query only the non-leader replicas, since we haven't opened a new searcher on the leader yet
    // Should be immediate
    waitForNumDocsInAllReplicas(4, getNonLeaderReplias(collectionName), 0);
    // Options are, wait or retry...
    for (int i = 0; i < 3; i++) {
        UpdateRequest ureq = new UpdateRequest().add(sdoc("id", "7"));
        ureq.setParam("collection", collectionName);
        ureq.setParam(UpdateRequest.MIN_REPFACT, "2");
        NamedList<Object> response = cloudClient.request(ureq);
        if ((Integer) ((NamedList<Object>) response.get("responseHeader")).get(UpdateRequest.REPFACT) >= 2) {
            break;
        }
        LOG.info("Min RF not achieved yet. retrying");
    }
    checkRTG(3, 7, cluster.getJettySolrRunners());
    DirectUpdateHandler2.commitOnClose = false;
    ChaosMonkey.stop(solrRunner);
    waitForState("Replica still up", collectionName, activeReplicaCount(0, 1, 0));
    DirectUpdateHandler2.commitOnClose = true;
    ChaosMonkey.start(solrRunner);
    waitForState("Replica didn't recover", collectionName, activeReplicaCount(0, 2, 0));
    checkRTG(3, 7, cluster.getJettySolrRunners());
    // Should be immediate
    waitForNumDocsInAllReplicas(5, getNonLeaderReplias(collectionName), 0);
    cluster.getSolrClient().commit(collectionName);
    // Test replica recovery apply buffer updates
    Semaphore waitingForBufferUpdates = new Semaphore(0);
    Semaphore waitingForReplay = new Semaphore(0);
    RecoveryStrategy.testing_beforeReplayBufferingUpdates = () -> {
        try {
            waitingForReplay.release();
            waitingForBufferUpdates.acquire();
        } catch (InterruptedException e) {
            e.printStackTrace();
            fail("Test interrupted: " + e.getMessage());
        }
    };
    if (useKill) {
        ChaosMonkey.kill(solrRunner);
    } else {
        ChaosMonkey.stop(solrRunner);
    }
    ChaosMonkey.start(solrRunner);
    waitingForReplay.acquire();
    new UpdateRequest().add(sdoc("id", "8")).add(sdoc("id", "9")).process(cloudClient, collectionName);
    waitingForBufferUpdates.release();
    RecoveryStrategy.testing_beforeReplayBufferingUpdates = null;
    waitForState("Replica didn't recover", collectionName, activeReplicaCount(0, 2, 0));
    checkRTG(3, 9, cluster.getJettySolrRunners());
    for (SolrCore solrCore : getSolrCore(false)) {
        RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
        assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
        iwRef.decref();
    }
}