Examples with UpdateCommand org.apache.solr.update.UpdateCommand used on opensource projects

Example 1 with UpdateCommand

use of org.apache.solr.update.UpdateCommand in project lucene-solr by apache.

the class DocExpirationUpdateProcessorFactoryTest method testAutomaticDeletes.

public void testAutomaticDeletes() throws Exception {
    // get a handle on our recorder
    UpdateRequestProcessorChain chain = h.getCore().getUpdateProcessingChain("scheduled-delete");
    assertNotNull(chain);
    List<UpdateRequestProcessorFactory> factories = chain.getProcessors();
    assertEquals("did number of processors configured in chain get changed?", 5, factories.size());
    assertTrue("Expected [1] RecordingUpdateProcessorFactory: " + factories.get(1).getClass(), factories.get(1) instanceof RecordingUpdateProcessorFactory);
    RecordingUpdateProcessorFactory recorder = (RecordingUpdateProcessorFactory) factories.get(1);
    try {
        recorder.startRecording();
        // more then one iter to verify it's recurring
        final int numItersToCheck = 1 + RANDOM_MULTIPLIER;
        for (int i = 0; i < numItersToCheck; i++) {
            UpdateCommand tmp;
            // be generous in how long we wait, some jenkins machines are slooooow
            tmp = recorder.commandQueue.poll(30, TimeUnit.SECONDS);
            // we can be confident in the order because DocExpirationUpdateProcessorFactory
            // uses the same request for both the delete & the commit -- and both 
            // RecordingUpdateProcessorFactory's getInstance & startRecording methods are 
            // synchronized.  So it should not be possible to start recording in the 
            // middle of the two commands
            assertTrue("expected DeleteUpdateCommand: " + tmp.getClass(), tmp instanceof DeleteUpdateCommand);
            DeleteUpdateCommand delete = (DeleteUpdateCommand) tmp;
            assertFalse(delete.isDeleteById());
            assertNotNull(delete.getQuery());
            assertTrue(delete.getQuery(), delete.getQuery().startsWith("{!cache=false}eXpField_tdt:[* TO "));
            // commit should be immediately after the delete
            tmp = recorder.commandQueue.poll(5, TimeUnit.SECONDS);
            assertTrue("expected CommitUpdateCommand: " + tmp.getClass(), tmp instanceof CommitUpdateCommand);
            CommitUpdateCommand commit = (CommitUpdateCommand) tmp;
            assertTrue(commit.softCommit);
            assertTrue(commit.openSearcher);
        }
    } finally {
        recorder.stopRecording();
    }
}

Example 2 with UpdateCommand

use of org.apache.solr.update.UpdateCommand in project lucene-solr by apache.

the class DistributedUpdateProcessor method versionAdd.

/**
   * @return whether or not to drop this cmd
   * @throws IOException If there is a low-level I/O error.
   */
protected boolean versionAdd(AddUpdateCommand cmd) throws IOException {
    BytesRef idBytes = cmd.getIndexedId();
    if (idBytes == null) {
        super.processAdd(cmd);
        return false;
    }
    if (vinfo == null) {
        if (AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Atomic document updates are not supported unless <updateLog/> is configured");
        } else {
            super.processAdd(cmd);
            return false;
        }
    }
    // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not use a pluggable hash here)
    int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
    // at this point, there is an update we need to try and apply.
    // we may or may not be the leader.
    // Find any existing version in the document
    // TODO: don't reuse update commands any more!
    long versionOnUpdate = cmd.getVersion();
    if (versionOnUpdate == 0) {
        SolrInputField versionField = cmd.getSolrInputDocument().getField(CommonParams.VERSION_FIELD);
        if (versionField != null) {
            Object o = versionField.getValue();
            versionOnUpdate = o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString());
        } else {
            // Find the version
            String versionOnUpdateS = req.getParams().get(CommonParams.VERSION_FIELD);
            versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
        }
    }
    boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
    boolean leaderLogic = isLeader && !isReplayOrPeersync;
    boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
    VersionBucket bucket = vinfo.bucket(bucketHash);
    long dependentVersionFound = -1;
    // this update depends), before entering the synchronized block
    if (!leaderLogic && cmd.isInPlaceUpdate()) {
        dependentVersionFound = waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync, bucket);
        if (dependentVersionFound == -1) {
            // it means the document has been deleted by now at the leader. drop this update
            return true;
        }
    }
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            //just in case anyone is waiting let them know that we have a new update
            bucket.notifyAll();
            // we obtain the version when synchronized and then do the add so we can ensure that
            // if version1 < version2 then version1 is actually added before version2.
            // even if we don't store the version field, synchronizing on the bucket
            // will enable us to know what version happened first, and thus enable
            // realtime-get to work reliably.
            // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?
            // there may be other reasons in the future for a version on the commands
            boolean checkDeleteByQueries = false;
            if (versionsStored) {
                long bucketVersion = bucket.highest;
                if (leaderLogic) {
                    if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {
                        // forwarded from a collection but we are not buffering so strip original version and apply our own
                        // see SOLR-5308
                        log.info("Removing version field from doc: " + cmd.getPrintableId());
                        cmd.solrDoc.remove(CommonParams.VERSION_FIELD);
                        versionOnUpdate = 0;
                    }
                    boolean updated = getUpdatedDocument(cmd, versionOnUpdate);
                    // leaders can also be in buffering state during "migrate" API call, see SOLR-5308
                    if (forwardedFromCollection && ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        log.info("Leader logic applied but update log is buffering: " + cmd.getPrintableId());
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (versionOnUpdate != 0) {
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        long foundVersion = lastVersion == null ? -1 : lastVersion;
                        if (versionOnUpdate == foundVersion || (versionOnUpdate < 0 && foundVersion < 0) || (versionOnUpdate == 1 && foundVersion > 0)) {
                        // we're ok if versions match, or if both are negative (all missing docs are equal), or if cmd
                        // specified it must exist (versionOnUpdate==1) and it does.
                        } else {
                            throw new SolrException(ErrorCode.CONFLICT, "version conflict for " + cmd.getPrintableId() + " expected=" + versionOnUpdate + " actual=" + foundVersion);
                        }
                    }
                    long version = vinfo.getNewClock();
                    cmd.setVersion(version);
                    cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version);
                    bucket.updateHighest(version);
                } else {
                    // The leader forwarded us this update.
                    cmd.setVersion(versionOnUpdate);
                    if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (cmd.isInPlaceUpdate()) {
                        long prev = cmd.prevVersion;
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        if (lastVersion == null || Math.abs(lastVersion) < prev) {
                            // this was checked for (in waitForDependentUpdates()) before entering the synchronized block.
                            // So we shouldn't be here, unless what must've happened is:
                            // by the time synchronization block was entered, the prev update was deleted by DBQ. Since
                            // now that update is not in index, the vinfo.lookupVersion() is possibly giving us a version 
                            // from the deleted list (which might be older than the prev update!) 
                            UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate);
                            if (fetchedFromLeader instanceof DeleteUpdateCommand) {
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, and the document" + " was deleted at the leader subsequently.", idBytes.utf8ToString());
                                versionDelete((DeleteUpdateCommand) fetchedFromLeader);
                                return true;
                            } else {
                                assert fetchedFromLeader instanceof AddUpdateCommand;
                                // Newer document was fetched from the leader. Apply that document instead of this current in-place update.
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}", idBytes.utf8ToString(), fetchedFromLeader);
                                // Make this update to become a non-inplace update containing the full document obtained from the leader
                                cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc;
                                cmd.prevVersion = -1;
                                cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD));
                                assert cmd.isInPlaceUpdate() == false;
                            }
                        } else {
                            if (lastVersion != null && Math.abs(lastVersion) > prev) {
                                // this means we got a newer full doc update and in that case it makes no sense to apply the older
                                // inplace update. Drop this update
                                log.info("Update was applied on version: " + prev + ", but last version I have is: " + lastVersion + ". Dropping current update.");
                                return true;
                            } else {
                                // We're good, we should apply this update. First, update the bucket's highest.
                                if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                                    bucket.updateHighest(versionOnUpdate);
                                }
                            }
                        }
                    }
                    if (!cmd.isInPlaceUpdate()) {
                        // if we aren't the leader, then we need to check that updates were not re-ordered
                        if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                            // we're OK... this update has a version higher than anything we've seen
                            // in this bucket so far, so we know that no reordering has yet occurred.
                            bucket.updateHighest(versionOnUpdate);
                        } else {
                            // there have been updates higher than the current update.  we need to check
                            // the specific version for this id.
                            Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                            if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
                                // This update is a repeat, or was reordered.  We need to drop this update.
                                log.debug("Dropping add update due to version {}", idBytes.utf8ToString());
                                return true;
                            }
                            // also need to re-apply newer deleteByQuery commands
                            checkDeleteByQueries = true;
                        }
                    }
                    if (replicaType == Replica.Type.TLOG && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);
                    }
                }
            }
            boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
            SolrInputDocument clonedDoc = null;
            if (willDistrib && cloneRequiredOnLeader) {
                clonedDoc = cmd.solrDoc.deepCopy();
            }
            // TODO: possibly set checkDeleteByQueries as a flag on the command?
            doLocalAdd(cmd);
            if (willDistrib && cloneRequiredOnLeader) {
                cmd.solrDoc = clonedDoc;
            }
        }
    // end synchronized (bucket)
    } finally {
        vinfo.unlockForUpdate();
    }
    return false;
}

Example 3 with UpdateCommand

use of org.apache.solr.update.UpdateCommand in project lucene-solr by apache.

the class DistributedUpdateProcessor method waitForDependentUpdates.

/**
   * This method checks the update/transaction logs and index to find out if the update ("previous update") that the current update
   * depends on (in the case that this current update is an in-place update) has already been completed. If not,
   * this method will wait for the missing update until it has arrived. If it doesn't arrive within a timeout threshold,
   * then this actively fetches from the leader.
   * 
   * @return -1 if the current in-place should be dropped, or last found version if previous update has been indexed.
   */
private long waitForDependentUpdates(AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync, VersionBucket bucket) throws IOException {
    long lastFoundVersion = 0;
    TimeOut waitTimeout = new TimeOut(5, TimeUnit.SECONDS);
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            Long lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());
            lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;
            if (Math.abs(lastFoundVersion) < cmd.prevVersion) {
                log.debug("Re-ordered inplace update. version={}, prevVersion={}, lastVersion={}, replayOrPeerSync={}, id={}", (cmd.getVersion() == 0 ? versionOnUpdate : cmd.getVersion()), cmd.prevVersion, lastFoundVersion, isReplayOrPeersync, cmd.getPrintableId());
            }
            while (Math.abs(lastFoundVersion) < cmd.prevVersion && !waitTimeout.hasTimedOut()) {
                try {
                    long timeLeft = waitTimeout.timeLeft(TimeUnit.MILLISECONDS);
                    if (timeLeft > 0) {
                        // wait(0) waits forever until notified, but we don't want that.
                        bucket.wait(timeLeft);
                    }
                } catch (InterruptedException ie) {
                    throw new RuntimeException(ie);
                }
                lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());
                lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;
            }
        }
    } finally {
        vinfo.unlockForUpdate();
    }
    if (Math.abs(lastFoundVersion) > cmd.prevVersion) {
        // we can drop the current update.
        if (log.isDebugEnabled()) {
            log.debug("Update was applied on version: {}, but last version I have is: {}" + ". Current update should be dropped. id={}", cmd.prevVersion, lastFoundVersion, cmd.getPrintableId());
        }
        return -1;
    } else if (Math.abs(lastFoundVersion) == cmd.prevVersion) {
        assert 0 < lastFoundVersion : "prevVersion " + cmd.prevVersion + " found but is a delete!";
        if (log.isDebugEnabled()) {
            log.debug("Dependent update found. id={}", cmd.getPrintableId());
        }
        return lastFoundVersion;
    }
    // We have waited enough, but dependent update didn't arrive. Its time to actively fetch it from leader
    log.info("Missing update, on which current in-place update depends on, hasn't arrived. id={}, looking for version={}, last found version={}", cmd.getPrintableId(), cmd.prevVersion, lastFoundVersion);
    UpdateCommand missingUpdate = fetchFullUpdateFromLeader(cmd, versionOnUpdate);
    if (missingUpdate instanceof DeleteUpdateCommand) {
        log.info("Tried to fetch document {} from the leader, but the leader says document has been deleted. " + "Deleting the document here and skipping this update: Last found version: {}, was looking for: {}", cmd.getPrintableId(), lastFoundVersion, cmd.prevVersion);
        versionDelete((DeleteUpdateCommand) missingUpdate);
        return -1;
    } else {
        assert missingUpdate instanceof AddUpdateCommand;
        log.debug("Fetched the document: {}", ((AddUpdateCommand) missingUpdate).getSolrInputDocument());
        versionAdd((AddUpdateCommand) missingUpdate);
        log.info("Added the fetched document, id=" + ((AddUpdateCommand) missingUpdate).getPrintableId() + ", version=" + missingUpdate.getVersion());
    }
    return missingUpdate.getVersion();
}