Examples with CommitUpdateCommand org.apache.solr.update.CommitUpdateCommand used on opensource projects

Example 1 with CommitUpdateCommand

use of org.apache.solr.update.CommitUpdateCommand in project lucene-solr by apache.

the class RequestHandlerUtils method handleCommit.

/**
   * Check the request parameters and decide if it should commit or optimize.
   * If it does, it will check other related parameters such as "waitFlush" and "waitSearcher"
   */
public static boolean handleCommit(SolrQueryRequest req, UpdateRequestProcessor processor, SolrParams params, boolean force) throws IOException {
    if (params == null) {
        params = new MapSolrParams(new HashMap<String, String>());
    }
    boolean optimize = params.getBool(UpdateParams.OPTIMIZE, false);
    boolean commit = params.getBool(UpdateParams.COMMIT, false);
    boolean softCommit = params.getBool(UpdateParams.SOFT_COMMIT, false);
    boolean prepareCommit = params.getBool(UpdateParams.PREPARE_COMMIT, false);
    if (optimize || commit || softCommit || prepareCommit || force) {
        CommitUpdateCommand cmd = new CommitUpdateCommand(req, optimize);
        updateCommit(cmd, params);
        processor.processCommit(cmd);
        return true;
    }
    return false;
}

Example 2 with CommitUpdateCommand

use of org.apache.solr.update.CommitUpdateCommand in project lucene-solr by apache.

the class ReplicateFromLeader method startReplication.

/**
   * Start a replication handler thread that will periodically pull indices from the shard leader
   * @param switchTransactionLog if true, ReplicationHandler will rotate the transaction log once
   * the replication is done
   */
public void startReplication(boolean switchTransactionLog) throws InterruptedException {
    try (SolrCore core = cc.getCore(coreName)) {
        if (core == null) {
            if (cc.isShutDown()) {
                return;
            } else {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "SolrCore not found:" + coreName + " in " + cc.getLoadedCoreNames());
            }
        }
        SolrConfig.UpdateHandlerInfo uinfo = core.getSolrConfig().getUpdateHandlerInfo();
        String pollIntervalStr = "00:00:03";
        if (uinfo.autoCommmitMaxTime != -1) {
            pollIntervalStr = toPollIntervalStr(uinfo.autoCommmitMaxTime / 2);
        } else if (uinfo.autoSoftCommmitMaxTime != -1) {
            pollIntervalStr = toPollIntervalStr(uinfo.autoSoftCommmitMaxTime / 2);
        }
        LOG.info("Will start replication from leader with poll interval: {}", pollIntervalStr);
        NamedList slaveConfig = new NamedList();
        slaveConfig.add("fetchFromLeader", true);
        slaveConfig.add("pollInterval", pollIntervalStr);
        NamedList replicationConfig = new NamedList();
        replicationConfig.add("slave", slaveConfig);
        String lastCommitVersion = getCommitVersion(core);
        if (lastCommitVersion != null) {
            lastVersion = Long.parseLong(lastCommitVersion);
        }
        replicationProcess = new ReplicationHandler();
        if (switchTransactionLog) {
            replicationProcess.setPollListener((solrCore, pollSuccess) -> {
                if (pollSuccess) {
                    String commitVersion = getCommitVersion(core);
                    if (commitVersion == null)
                        return;
                    if (Long.parseLong(commitVersion) == lastVersion)
                        return;
                    UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
                    SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
                    CommitUpdateCommand cuc = new CommitUpdateCommand(req, false);
                    cuc.setVersion(Long.parseLong(commitVersion));
                    updateLog.copyOverOldUpdates(cuc);
                    lastVersion = Long.parseLong(commitVersion);
                }
            });
        }
        replicationProcess.init(replicationConfig);
        replicationProcess.inform(core);
    }
}

Example 3 with CommitUpdateCommand

use of org.apache.solr.update.CommitUpdateCommand in project lucene-solr by apache.

the class RecoveryStrategy method replay.

private final Future<RecoveryInfo> replay(SolrCore core) throws InterruptedException, ExecutionException {
    if (testing_beforeReplayBufferingUpdates != null) {
        testing_beforeReplayBufferingUpdates.run();
    }
    if (replicaType == Replica.Type.TLOG) {
        // roll over all updates during buffering to new tlog, make RTG available
        SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
        core.getUpdateHandler().getUpdateLog().copyOverBufferingUpdates(new CommitUpdateCommand(req, false));
        return null;
    }
    Future<RecoveryInfo> future = core.getUpdateHandler().getUpdateLog().applyBufferedUpdates();
    if (future == null) {
        // no replay needed\
        LOG.info("No replay needed.");
    } else {
        LOG.info("Replaying buffered documents.");
        // wait for replay
        RecoveryInfo report = future.get();
        if (report.failed) {
            SolrException.log(LOG, "Replay failed");
            throw new SolrException(ErrorCode.SERVER_ERROR, "Replay failed");
        }
    }
    // solrcloud_debug
    cloudDebugLog(core, "replayed");
    return future;
}

Example 4 with CommitUpdateCommand

use of org.apache.solr.update.CommitUpdateCommand in project lucene-solr by apache.

the class PrepRecoveryOp method execute.

@Override
public void execute(CallInfo it) throws Exception {
    assert TestInjection.injectPrepRecoveryOpPauseForever();
    final SolrParams params = it.req.getParams();
    String cname = params.get(CoreAdminParams.CORE);
    if (cname == null) {
        cname = "";
    }
    String nodeName = params.get("nodeName");
    String coreNodeName = params.get("coreNodeName");
    Replica.State waitForState = Replica.State.getState(params.get(ZkStateReader.STATE_PROP));
    Boolean checkLive = params.getBool("checkLive");
    Boolean onlyIfLeader = params.getBool("onlyIfLeader");
    Boolean onlyIfLeaderActive = params.getBool("onlyIfLeaderActive");
    CoreContainer coreContainer = it.handler.coreContainer;
    // wait long enough for the leader conflict to work itself out plus a little extra
    int conflictWaitMs = coreContainer.getZkController().getLeaderConflictResolveWait();
    int maxTries = (int) Math.round(conflictWaitMs / 1000) + 3;
    log.info("Going to wait for coreNodeName: {}, state: {}, checkLive: {}, onlyIfLeader: {}, onlyIfLeaderActive: {}, maxTime: {} s", coreNodeName, waitForState, checkLive, onlyIfLeader, onlyIfLeaderActive, maxTries);
    Replica.State state = null;
    boolean live = false;
    int retry = 0;
    while (true) {
        try (SolrCore core = coreContainer.getCore(cname)) {
            if (core == null && retry == Math.min(30, maxTries)) {
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "core not found:" + cname);
            }
            if (core != null) {
                if (onlyIfLeader != null && onlyIfLeader) {
                    if (!core.getCoreDescriptor().getCloudDescriptor().isLeader()) {
                        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "We are not the leader");
                    }
                }
                // wait until we are sure the recovering node is ready
                // to accept updates
                CloudDescriptor cloudDescriptor = core.getCoreDescriptor().getCloudDescriptor();
                String collectionName = cloudDescriptor.getCollectionName();
                if (retry % 15 == 0) {
                    if (retry > 0 && log.isInfoEnabled())
                        log.info("After " + retry + " seconds, core " + cname + " (" + cloudDescriptor.getShardId() + " of " + cloudDescriptor.getCollectionName() + ") still does not have state: " + waitForState + "; forcing ClusterState update from ZooKeeper");
                    // force a cluster state update
                    coreContainer.getZkController().getZkStateReader().forceUpdateCollection(collectionName);
                }
                ClusterState clusterState = coreContainer.getZkController().getClusterState();
                DocCollection collection = clusterState.getCollection(collectionName);
                Slice slice = collection.getSlice(cloudDescriptor.getShardId());
                if (slice != null) {
                    final Replica replica = slice.getReplicasMap().get(coreNodeName);
                    if (replica != null) {
                        state = replica.getState();
                        live = clusterState.liveNodesContain(nodeName);
                        final Replica.State localState = cloudDescriptor.getLastPublished();
                        // TODO: This is funky but I've seen this in testing where the replica asks the
                        // leader to be in recovery? Need to track down how that happens ... in the meantime,
                        // this is a safeguard
                        boolean leaderDoesNotNeedRecovery = (onlyIfLeader != null && onlyIfLeader && core.getName().equals(replica.getStr("core")) && waitForState == Replica.State.RECOVERING && localState == Replica.State.ACTIVE && state == Replica.State.ACTIVE);
                        if (leaderDoesNotNeedRecovery) {
                            log.warn("Leader " + core.getName() + " ignoring request to be in the recovering state because it is live and active.");
                        }
                        boolean onlyIfActiveCheckResult = onlyIfLeaderActive != null && onlyIfLeaderActive && localState != Replica.State.ACTIVE;
                        log.info("In WaitForState(" + waitForState + "): collection=" + collectionName + ", shard=" + slice.getName() + ", thisCore=" + core.getName() + ", leaderDoesNotNeedRecovery=" + leaderDoesNotNeedRecovery + ", isLeader? " + core.getCoreDescriptor().getCloudDescriptor().isLeader() + ", live=" + live + ", checkLive=" + checkLive + ", currentState=" + state.toString() + ", localState=" + localState + ", nodeName=" + nodeName + ", coreNodeName=" + coreNodeName + ", onlyIfActiveCheckResult=" + onlyIfActiveCheckResult + ", nodeProps: " + replica);
                        if (!onlyIfActiveCheckResult && replica != null && (state == waitForState || leaderDoesNotNeedRecovery)) {
                            if (checkLive == null) {
                                break;
                            } else if (checkLive && live) {
                                break;
                            } else if (!checkLive && !live) {
                                break;
                            }
                        }
                    }
                }
            }
            if (retry++ == maxTries) {
                String collection = null;
                String leaderInfo = null;
                String shardId = null;
                try {
                    CloudDescriptor cloudDescriptor = core.getCoreDescriptor().getCloudDescriptor();
                    collection = cloudDescriptor.getCollectionName();
                    shardId = cloudDescriptor.getShardId();
                    leaderInfo = coreContainer.getZkController().getZkStateReader().getLeaderUrl(collection, shardId, 5000);
                } catch (Exception exc) {
                    leaderInfo = "Not available due to: " + exc;
                }
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "I was asked to wait on state " + waitForState + " for " + shardId + " in " + collection + " on " + nodeName + " but I still do not see the requested state. I see state: " + Objects.toString(state) + " live:" + live + " leader from ZK: " + leaderInfo);
            }
            if (coreContainer.isShutDown()) {
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Solr is shutting down");
            }
            // solrcloud_debug
            if (log.isDebugEnabled() && core != null) {
                try {
                    LocalSolrQueryRequest r = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
                    CommitUpdateCommand commitCmd = new CommitUpdateCommand(r, false);
                    commitCmd.softCommit = true;
                    core.getUpdateHandler().commit(commitCmd);
                    RefCounted<SolrIndexSearcher> searchHolder = core.getNewestSearcher(false);
                    SolrIndexSearcher searcher = searchHolder.get();
                    try {
                        log.debug(core.getCoreContainer().getZkController().getNodeName() + " to replicate " + searcher.search(new MatchAllDocsQuery(), 1).totalHits + " gen:" + core.getDeletionPolicy().getLatestCommit().getGeneration() + " data:" + core.getDataDir());
                    } finally {
                        searchHolder.decref();
                    }
                } catch (Exception e) {
                    log.debug("Error in solrcloud_debug block", e);
                }
            }
        }
        Thread.sleep(1000);
    }
    log.info("Waited coreNodeName: " + coreNodeName + ", state: " + waitForState + ", checkLive: " + checkLive + ", onlyIfLeader: " + onlyIfLeader + " for: " + retry + " seconds.");
}

Example 5 with CommitUpdateCommand

use of org.apache.solr.update.CommitUpdateCommand in project lucene-solr by apache.

the class JsonLoaderTest method testParsing.

public void testParsing() throws Exception {
    SolrQueryRequest req = req();
    SolrQueryResponse rsp = new SolrQueryResponse();
    BufferingRequestProcessor p = new BufferingRequestProcessor(null);
    JsonLoader loader = new JsonLoader();
    loader.load(req, rsp, new ContentStreamBase.StringStream(input), p);
    assertEquals(2, p.addCommands.size());
    AddUpdateCommand add = p.addCommands.get(0);
    SolrInputDocument d = add.solrDoc;
    SolrInputField f = d.getField("boosted");
    assertEquals(2, f.getValues().size());
    // 
    add = p.addCommands.get(1);
    d = add.solrDoc;
    f = d.getField("f1");
    assertEquals(2, f.getValues().size());
    assertEquals(false, add.overwrite);
    assertEquals(0, d.getField("f2").getValueCount());
    // parse the commit commands
    assertEquals(2, p.commitCommands.size());
    CommitUpdateCommand commit = p.commitCommands.get(0);
    assertFalse(commit.optimize);
    assertTrue(commit.waitSearcher);
    assertTrue(commit.openSearcher);
    commit = p.commitCommands.get(1);
    assertTrue(commit.optimize);
    assertFalse(commit.waitSearcher);
    assertFalse(commit.openSearcher);
    // DELETE COMMANDS
    assertEquals(4, p.deleteCommands.size());
    DeleteUpdateCommand delete = p.deleteCommands.get(0);
    assertEquals(delete.id, "ID");
    assertEquals(delete.query, null);
    assertEquals(delete.commitWithin, -1);
    delete = p.deleteCommands.get(1);
    assertEquals(delete.id, "ID");
    assertEquals(delete.query, null);
    assertEquals(delete.commitWithin, 500);
    delete = p.deleteCommands.get(2);
    assertEquals(delete.id, null);
    assertEquals(delete.query, "QUERY");
    assertEquals(delete.commitWithin, -1);
    delete = p.deleteCommands.get(3);
    assertEquals(delete.id, null);
    assertEquals(delete.query, "QUERY");
    assertEquals(delete.commitWithin, 500);
    // ROLLBACK COMMANDS
    assertEquals(1, p.rollbackCommands.size());
    req.close();
}