Examples with ConcurrentUpdateSolrClient org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient used on opensource projects

Example 11 with ConcurrentUpdateSolrClient

use of org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient in project mycore by MyCoRe-Org.

the class MCRSolrCommands method createIndex.

@MCRCommand(syntax = "create solr metadata and content index at {0}", help = "create solr's metadata and content index on specific solr server core", order = 120)
public static void createIndex(String url) throws Exception {
    MCRSolrCore core = new MCRSolrCore(url);
    SolrClient concurrentSolrClient = core.getConcurrentClient();
    SolrClient solrClient = core.getClient();
    MCRSolrIndexer.rebuildMetadataIndex(concurrentSolrClient);
    MCRSolrIndexer.rebuildContentIndex(solrClient);
    if (concurrentSolrClient instanceof ConcurrentUpdateSolrClient) {
        ((ConcurrentUpdateSolrClient) concurrentSolrClient).blockUntilFinished();
    }
    solrClient.optimize();
}

Example 12 with ConcurrentUpdateSolrClient

use of org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient in project mycore by MyCoRe-Org.

the class MCRSolrInputDocumentsHandler method index.

/* (non-Javadoc)
     * @see org.mycore.solr.index.handlers.MCRSolrAbstractIndexHandler#index()
     */
@Override
public void index() throws IOException, SolrServerException {
    if (documents == null || documents.isEmpty()) {
        LOGGER.warn("No input documents to index.");
        return;
    }
    int totalCount = documents.size();
    LOGGER.info("Handling {} documents", totalCount);
    SolrClient solrClient = getSolrClient();
    if (solrClient instanceof ConcurrentUpdateSolrClient) {
        LOGGER.info("Detected ConcurrentUpdateSolrClient. Split up batch update.");
        splitDocuments();
        // for statistics:
        documents.clear();
        return;
    }
    UpdateResponse updateResponse;
    try {
        UpdateRequest updateRequest = getUpdateRequest(MCRSolrConstants.UPDATE_PATH);
        updateRequest.add(documents);
        updateResponse = updateRequest.process(getSolrClient());
    } catch (Throwable e) {
        LOGGER.warn("Error while indexing document collection. Split and retry.");
        splitDocuments();
        return;
    }
    if (updateResponse.getStatus() != 0) {
        LOGGER.error("Error while indexing document collection. Split and retry: {}", updateResponse.getResponse());
        splitDocuments();
    } else {
        LOGGER.info("Sending {} documents was successful in {} ms.", totalCount, updateResponse.getElapsedTime());
    }
}

Example 13 with ConcurrentUpdateSolrClient

use of org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient in project gora by apache.

the class SolrStore method initialize.

/**
 * Initialize the data store by reading the credentials, setting the client's properties up and
 * reading the mapping file. Initialize is called when then the call to
 * {@link org.apache.gora.store.DataStoreFactory#createDataStore} is made.
 *
 * @param keyClass
 * @param persistentClass
 * @param properties
 */
@Override
public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) throws GoraException {
    super.initialize(keyClass, persistentClass, properties);
    try {
        String mappingFile = DataStoreFactory.getMappingFile(properties, this, DEFAULT_MAPPING_FILE);
        mapping = readMapping(mappingFile);
    } catch (IOException e) {
        throw new GoraException(e);
    }
    String queueSizeString = DataStoreFactory.findProperty(properties, this, SOLR_CONCURRENT_CLIENT_QUEUE_SIZE_PROPERTY, null);
    if (queueSizeString != null) {
        try {
            queueSize = Integer.parseInt(queueSizeString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid concurrent client queue size '{}' , using default {}", queueSizeString, DEFAULT_SOLR_CONCURRENT_CLIENT_QUEUE_SIZE);
        }
    }
    String threadCountString = DataStoreFactory.findProperty(properties, this, SOLR_CONCURRENT_CLIENT_THREAD_COUNT_PROPERTY, null);
    if (threadCountString != null) {
        try {
            threadCount = Integer.parseInt(threadCountString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid concurrent client thread count '{}' , using default {}", threadCountString, DEFAULT_SOLR_CONCURRENT_CLIENT_THREAD_COUNT);
        }
    }
    SolrClientUrl = DataStoreFactory.findProperty(properties, this, SOLR_URL_PROPERTY, null);
    solrConfig = DataStoreFactory.findProperty(properties, this, SOLR_CONFIG_PROPERTY, null);
    solrSchema = DataStoreFactory.findProperty(properties, this, SOLR_SCHEMA_PROPERTY, null);
    solrJServerImpl = DataStoreFactory.findProperty(properties, this, SOLR_SOLRJSERVER_IMPL, "http");
    serverUserAuth = DataStoreFactory.findBooleanProperty(properties, this, SOLR_SERVER_USER_AUTH, "false");
    if (serverUserAuth) {
        serverUsername = DataStoreFactory.findProperty(properties, this, SOLR_SERVER_USERNAME, null);
        serverPassword = DataStoreFactory.findProperty(properties, this, SOLR_SERVER_PASSWORD, null);
    }
    LOG.info("Using Solr server at " + SolrClientUrl);
    String solrJServerType = ((solrJServerImpl == null || solrJServerImpl.equals("")) ? "http" : solrJServerImpl);
    // HttpSolrClient - denoted by "http" in properties
    if (solrJServerType.toLowerCase(Locale.getDefault()).equals("http")) {
        LOG.info("Using HttpSolrClient Solrj implementation.");
        if (serverUserAuth) {
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.set("httpBasicAuthUser", serverUsername);
            params.set("httpBasicAuthPassword", serverPassword);
            this.adminServer = new HttpSolrClient.Builder(SolrClientUrl).withHttpClient(HttpClientUtil.createClient(params)).build();
            this.server = new HttpSolrClient.Builder(SolrClientUrl + "/" + mapping.getCoreName()).withHttpClient(HttpClientUtil.createClient(params)).build();
        } else {
            this.adminServer = new HttpSolrClient.Builder(SolrClientUrl).build();
            this.server = new HttpSolrClient.Builder(SolrClientUrl + "/" + mapping.getCoreName()).build();
        }
    // CloudSolrClient - denoted by "cloud" in properties
    } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("cloud")) {
        LOG.info("Using CloudSolrClient Solrj implementation.");
        if (serverUserAuth) {
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.set("httpBasicAuthUser", serverUsername);
            params.set("httpBasicAuthPassword", serverPassword);
            List<String> adminSolrUrls = new ArrayList();
            adminSolrUrls.add(SolrClientUrl);
            this.adminServer = new CloudSolrClient.Builder(adminSolrUrls).withHttpClient(HttpClientUtil.createClient(params)).build();
            List<String> serverSolrUrls = new ArrayList();
            serverSolrUrls.add(SolrClientUrl + "/" + mapping.getCoreName());
            this.server = new CloudSolrClient.Builder(serverSolrUrls).withHttpClient(HttpClientUtil.createClient(params)).build();
        } else {
            List<String> adminSolrUrls = new ArrayList();
            adminSolrUrls.add(SolrClientUrl);
            this.adminServer = new CloudSolrClient.Builder(adminSolrUrls).build();
            List<String> serverSolrUrls = new ArrayList();
            serverSolrUrls.add(SolrClientUrl + "/" + mapping.getCoreName());
            this.server = new CloudSolrClient.Builder(serverSolrUrls).build();
        }
    } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("concurrent")) {
        LOG.info("Using ConcurrentUpdateSolrClient Solrj implementation.");
        // LBHttpSolrClient - denoted by "loadbalance" in properties
        if (serverUserAuth) {
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.set("httpBasicAuthUser", serverUsername);
            params.set("httpBasicAuthPassword", serverPassword);
            this.adminServer = new ConcurrentUpdateSolrClient.Builder(SolrClientUrl).withHttpClient(HttpClientUtil.createClient(params)).withQueueSize(queueSize).withThreadCount(threadCount).build();
            this.server = new ConcurrentUpdateSolrClient.Builder(SolrClientUrl + "/" + mapping.getCoreName()).withHttpClient(HttpClientUtil.createClient(params)).withQueueSize(queueSize).withThreadCount(threadCount).build();
        } else {
            this.adminServer = new ConcurrentUpdateSolrClient.Builder(SolrClientUrl).withQueueSize(queueSize).withThreadCount(threadCount).build();
            this.server = new ConcurrentUpdateSolrClient.Builder(SolrClientUrl + "/" + mapping.getCoreName()).withQueueSize(queueSize).withThreadCount(threadCount).build();
        }
    } else if (solrJServerType.toLowerCase(Locale.getDefault()).equals("loadbalance")) {
        LOG.info("Using LBHttpSolrClient Solrj implementation.");
        if (serverUserAuth) {
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.set("httpBasicAuthUser", serverUsername);
            params.set("httpBasicAuthPassword", serverPassword);
            String[] solrUrlElements = StringUtils.split(SolrClientUrl);
            this.adminServer = new LBHttpSolrClient.Builder().withBaseSolrUrls(solrUrlElements).withHttpClient(HttpClientUtil.createClient(params)).build();
            if (solrUrlElements.length > 0) {
                for (int counter = 0; counter < solrUrlElements.length; counter++) {
                    solrUrlElements[counter] = solrUrlElements[counter] + "/" + mapping.getCoreName();
                }
            }
            this.server = new LBHttpSolrClient.Builder().withHttpClient(HttpClientUtil.createClient(params)).withBaseSolrUrls(solrUrlElements).build();
        } else {
            String[] solrUrlElements = StringUtils.split(SolrClientUrl);
            this.adminServer = new LBHttpSolrClient.Builder().withBaseSolrUrls(solrUrlElements).build();
            if (solrUrlElements.length > 0) {
                for (int counter = 0; counter < solrUrlElements.length; counter++) {
                    solrUrlElements[counter] = solrUrlElements[counter] + "/" + mapping.getCoreName();
                }
            }
            this.server = new LBHttpSolrClient.Builder().withBaseSolrUrls(solrUrlElements).build();
        }
    }
    if (autoCreateSchema) {
        createSchema();
    }
    String batchSizeString = DataStoreFactory.findProperty(properties, this, SOLR_BATCH_SIZE_PROPERTY, null);
    if (batchSizeString != null) {
        try {
            batchSize = Integer.parseInt(batchSizeString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid batch size '{}', using default {}", batchSizeString, DEFAULT_BATCH_SIZE);
        }
    }
    batch = new ArrayList<>(batchSize);
    String commitWithinString = DataStoreFactory.findProperty(properties, this, SOLR_COMMIT_WITHIN_PROPERTY, null);
    if (commitWithinString != null) {
        try {
            commitWithin = Integer.parseInt(commitWithinString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid commit within '{}' , using default {}", commitWithinString, DEFAULT_COMMIT_WITHIN);
        }
    }
    String resultsSizeString = DataStoreFactory.findProperty(properties, this, SOLR_RESULTS_SIZE_PROPERTY, null);
    if (resultsSizeString != null) {
        try {
            resultsSize = Integer.parseInt(resultsSizeString);
        } catch (NumberFormatException nfe) {
            LOG.warn("Invalid results size '{}' , using default {}", resultsSizeString, DEFAULT_RESULTS_SIZE);
        }
    }
}