Examples with ModifiableSolrParams org.apache.solr.common.params.ModifiableSolrParams used on opensource projects

Example 31 with ModifiableSolrParams

use of org.apache.solr.common.params.ModifiableSolrParams in project lucene-solr by apache.

the class CdcrUpdateLog method copyBufferedUpdates.

/**
   * <p>
   *   Read the entries from the given tlog file and replay them as buffered updates.
   *   The buffered tlog that we are trying to copy might contain duplicate operations with the
   *   current update log. During the tlog replication process, the replica might buffer update operations
   *   that will be present also in the tlog files downloaded from the leader. In order to remove these
   *   duplicates, it will skip any operations with a version inferior to the latest know version.
   */
private void copyBufferedUpdates(File tlogSrc, long offsetSrc, long latestVersion) {
    recoveryInfo = new RecoveryInfo();
    recoveryInfo.positionOfStart = tlog == null ? 0 : tlog.snapshot();
    state = State.BUFFERING;
    operationFlags |= FLAG_GAP;
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM, DistributedUpdateProcessor.DistribPhase.FROMLEADER.toString());
    SolrQueryRequest req = new LocalSolrQueryRequest(uhandler.core, params);
    CdcrTransactionLog src = new CdcrTransactionLog(tlogSrc, null, true);
    TransactionLog.LogReader tlogReader = src.getReader(offsetSrc);
    try {
        int operationAndFlags = 0;
        for (; ; ) {
            Object o = tlogReader.next();
            // we reached the end of the tlog
            if (o == null)
                break;
            // should currently be a List<Oper,Ver,Doc/Id>
            List entry = (List) o;
            operationAndFlags = (Integer) entry.get(0);
            int oper = operationAndFlags & OPERATION_MASK;
            long version = (Long) entry.get(1);
            if (version <= latestVersion) {
                // probably a buffered update that is also present in a tlog file coming from the leader,
                // skip it.
                log.debug("Dropping buffered operation - version {} < {}", version, latestVersion);
                continue;
            }
            switch(oper) {
                case UpdateLog.ADD:
                    {
                        SolrInputDocument sdoc = (SolrInputDocument) entry.get(entry.size() - 1);
                        AddUpdateCommand cmd = new AddUpdateCommand(req);
                        cmd.solrDoc = sdoc;
                        cmd.setVersion(version);
                        cmd.setFlags(UpdateCommand.BUFFERING);
                        this.add(cmd);
                        break;
                    }
                case UpdateLog.DELETE:
                    {
                        byte[] idBytes = (byte[]) entry.get(2);
                        DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
                        cmd.setIndexedId(new BytesRef(idBytes));
                        cmd.setVersion(version);
                        cmd.setFlags(UpdateCommand.BUFFERING);
                        this.delete(cmd);
                        break;
                    }
                case UpdateLog.DELETE_BY_QUERY:
                    {
                        String query = (String) entry.get(2);
                        DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
                        cmd.query = query;
                        cmd.setVersion(version);
                        cmd.setFlags(UpdateCommand.BUFFERING);
                        this.deleteByQuery(cmd);
                        break;
                    }
                default:
                    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid Operation! " + oper);
            }
        }
    } catch (Exception e) {
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to copy buffered updates", e);
    } finally {
        try {
            tlogReader.close();
        } finally {
            this.doClose(src);
        }
    }
}

Example 32 with ModifiableSolrParams

use of org.apache.solr.common.params.ModifiableSolrParams in project lucene-solr by apache.

the class PeerSync method requestFingerprint.

private void requestFingerprint(String replica) {
    SyncShardRequest sreq = new SyncShardRequest();
    requests.add(sreq);
    sreq.shards = new String[] { replica };
    sreq.actualShards = sreq.shards;
    sreq.params = new ModifiableSolrParams();
    sreq.params = new ModifiableSolrParams();
    sreq.params.set("qt", "/get");
    sreq.params.set(DISTRIB, false);
    sreq.params.set("getFingerprint", String.valueOf(Long.MAX_VALUE));
    shardHandler.submit(sreq, replica, sreq.params);
}

Example 33 with ModifiableSolrParams

use of org.apache.solr.common.params.ModifiableSolrParams in project lucene-solr by apache.

the class TestRandomFaceting method getIndexSortedAllFacetValues.

private String getIndexSortedAllFacetValues(ModifiableSolrParams in, List<String> methods) throws Exception {
    ModifiableSolrParams params = new ModifiableSolrParams(in);
    params.set("facet.sort", "index");
    String goodOldMethod = methods.get(random().nextInt(methods.size()));
    params.set("facet.method", goodOldMethod);
    params.set("facet.exists", "false");
    if (random().nextBoolean()) {
        params.remove("facet.exists");
    }
    params.set("facet.limit", -1);
    params.set("facet.offset", 0);
    final String query;
    SolrQueryRequest req = null;
    try {
        req = req(params);
        query = h.query(req);
    } finally {
        req.close();
    }
    return query;
}

Example 34 with ModifiableSolrParams

use of org.apache.solr.common.params.ModifiableSolrParams in project lucene-solr by apache.

the class BasicDistributedZk2Test method brindDownShardIndexSomeDocsAndRecover.

private void brindDownShardIndexSomeDocsAndRecover() throws Exception {
    SolrQuery query = new SolrQuery("*:*");
    query.set("distrib", false);
    commit();
    long deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound();
    query("q", "*:*", "sort", "n_tl1 desc");
    int oldLiveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
    assertEquals(5, oldLiveNodes);
    // kill a shard
    CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
    // ensure shard is dead
    try {
        index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!");
        fail("This server should be down and this update should have failed");
    } catch (SolrServerException e) {
    // expected..
    }
    commit();
    query("q", "*:*", "sort", "n_tl1 desc");
    // long cloudClientDocs = cloudClient.query(new
    // SolrQuery("*:*")).getResults().getNumFound();
    // System.out.println("clouddocs:" + cloudClientDocs);
    // try to index to a living shard at shard2
    long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
    cloudClient.getZkStateReader().getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000);
    index_specific(shardToJetty.get(SHARD1).get(1).client.solrClient, id, 1000, i1, 108, t1, "specific doc!");
    commit();
    checkShardConsistency(true, false);
    query("q", "*:*", "sort", "n_tl1 desc");
    cloudClient.setDefaultCollection(DEFAULT_COLLECTION);
    long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(numFound1 + 1, numFound2);
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", 1001);
    controlClient.add(doc);
    // try adding a doc with CloudSolrServer
    UpdateRequest ureq = new UpdateRequest();
    ureq.add(doc);
    try {
        ureq.process(cloudClient);
    } catch (SolrServerException e) {
        // try again
        Thread.sleep(3500);
        ureq.process(cloudClient);
    }
    commit();
    query("q", "*:*", "sort", "n_tl1 desc");
    long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
    // lets just check that the one doc since last commit made it in...
    assertEquals(numFound2 + 1, numFound3);
    // test debugging
    testDebugQueries();
    if (VERBOSE) {
        System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound());
        for (SolrClient client : clients) {
            try {
                SolrQuery q = new SolrQuery("*:*");
                q.set("distrib", false);
                System.err.println(client.query(q).getResults().getNumFound());
            } catch (Exception e) {
            }
        }
    }
    // TODO: This test currently fails because debug info is obtained only
    // on shards with matches.
    // query("q","matchesnothing","fl","*,score", "debugQuery", "true");
    // this should trigger a recovery phase on deadShard
    ChaosMonkey.start(deadShard.jetty);
    // make sure we have published we are recovering
    Thread.sleep(1500);
    waitForRecoveriesToFinish(false);
    deadShardCount = shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound();
    // if we properly recovered, we should now have the couple missing docs that
    // came in while shard was down
    checkShardConsistency(true, false);
    // recover over 100 docs so we do more than just peer sync (replicate recovery)
    chaosMonkey.stopJetty(deadShard);
    for (int i = 0; i < 226; i++) {
        doc = new SolrInputDocument();
        doc.addField("id", 2000 + i);
        controlClient.add(doc);
        ureq = new UpdateRequest();
        ureq.add(doc);
        // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN);
        ureq.process(cloudClient);
    }
    commit();
    Thread.sleep(1500);
    ChaosMonkey.start(deadShard.jetty);
    // make sure we have published we are recovering
    Thread.sleep(1500);
    waitForThingsToLevelOut(60);
    Thread.sleep(500);
    waitForRecoveriesToFinish(false);
    checkShardConsistency(true, false);
    // try a backup command
    try (final HttpSolrClient client = getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) {
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set("qt", ReplicationHandler.PATH);
        params.set("command", "backup");
        Path location = createTempDir();
        location = FilterPath.unwrap(location).toRealPath();
        params.set("location", location.toString());
        QueryRequest request = new QueryRequest(params);
        client.request(request, DEFAULT_TEST_COLLECTION_NAME);
        checkForBackupSuccess(client, location);
        client.close();
    }
}

Example 35 with ModifiableSolrParams

use of org.apache.solr.common.params.ModifiableSolrParams in project lucene-solr by apache.

the class BasicDistributedZkTest method createCollection.

protected CollectionAdminResponse createCollection(Map<String, List<Integer>> collectionInfos, String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrClient client, String createNodeSetStr) throws SolrServerException, IOException {
    // TODO: Use CollectionAdminRequest for this test
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("action", CollectionAction.CREATE.toString());
    params.set(OverseerCollectionMessageHandler.NUM_SLICES, numShards);
    params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas);
    params.set(ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode);
    if (createNodeSetStr != null)
        params.set(OverseerCollectionMessageHandler.CREATE_NODE_SET, createNodeSetStr);
    int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0;
    List<Integer> list = new ArrayList<>();
    list.add(numShards);
    list.add(numReplicas);
    if (collectionInfos != null) {
        collectionInfos.put(collectionName, list);
    }
    params.set("name", collectionName);
    SolrRequest request = new QueryRequest(params);
    request.setPath("/admin/collections");
    CollectionAdminResponse res = new CollectionAdminResponse();
    if (client == null) {
        final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL().substring(0, ((HttpSolrClient) clients.get(clientIndex)).getBaseURL().length() - DEFAULT_COLLECTION.length() - 1);
        try (SolrClient aClient = createNewSolrClient("", baseUrl)) {
            res.setResponse(aClient.request(request));
        }
    } else {
        res.setResponse(client.request(request));
    }
    return res;
}