Examples with VersionBucket org.apache.solr.update.VersionBucket used on opensource projects

Example 1 with VersionBucket

use of org.apache.solr.update.VersionBucket in project lucene-solr by apache.

the class DistributedUpdateProcessor method versionAdd.

/**
   * @return whether or not to drop this cmd
   * @throws IOException If there is a low-level I/O error.
   */
protected boolean versionAdd(AddUpdateCommand cmd) throws IOException {
    BytesRef idBytes = cmd.getIndexedId();
    if (idBytes == null) {
        super.processAdd(cmd);
        return false;
    }
    if (vinfo == null) {
        if (AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Atomic document updates are not supported unless <updateLog/> is configured");
        } else {
            super.processAdd(cmd);
            return false;
        }
    }
    // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not use a pluggable hash here)
    int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
    // at this point, there is an update we need to try and apply.
    // we may or may not be the leader.
    // Find any existing version in the document
    // TODO: don't reuse update commands any more!
    long versionOnUpdate = cmd.getVersion();
    if (versionOnUpdate == 0) {
        SolrInputField versionField = cmd.getSolrInputDocument().getField(CommonParams.VERSION_FIELD);
        if (versionField != null) {
            Object o = versionField.getValue();
            versionOnUpdate = o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString());
        } else {
            // Find the version
            String versionOnUpdateS = req.getParams().get(CommonParams.VERSION_FIELD);
            versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
        }
    }
    boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
    boolean leaderLogic = isLeader && !isReplayOrPeersync;
    boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
    VersionBucket bucket = vinfo.bucket(bucketHash);
    long dependentVersionFound = -1;
    // this update depends), before entering the synchronized block
    if (!leaderLogic && cmd.isInPlaceUpdate()) {
        dependentVersionFound = waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync, bucket);
        if (dependentVersionFound == -1) {
            // it means the document has been deleted by now at the leader. drop this update
            return true;
        }
    }
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            //just in case anyone is waiting let them know that we have a new update
            bucket.notifyAll();
            // we obtain the version when synchronized and then do the add so we can ensure that
            // if version1 < version2 then version1 is actually added before version2.
            // even if we don't store the version field, synchronizing on the bucket
            // will enable us to know what version happened first, and thus enable
            // realtime-get to work reliably.
            // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?
            // there may be other reasons in the future for a version on the commands
            boolean checkDeleteByQueries = false;
            if (versionsStored) {
                long bucketVersion = bucket.highest;
                if (leaderLogic) {
                    if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {
                        // forwarded from a collection but we are not buffering so strip original version and apply our own
                        // see SOLR-5308
                        log.info("Removing version field from doc: " + cmd.getPrintableId());
                        cmd.solrDoc.remove(CommonParams.VERSION_FIELD);
                        versionOnUpdate = 0;
                    }
                    boolean updated = getUpdatedDocument(cmd, versionOnUpdate);
                    // leaders can also be in buffering state during "migrate" API call, see SOLR-5308
                    if (forwardedFromCollection && ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        log.info("Leader logic applied but update log is buffering: " + cmd.getPrintableId());
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (versionOnUpdate != 0) {
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        long foundVersion = lastVersion == null ? -1 : lastVersion;
                        if (versionOnUpdate == foundVersion || (versionOnUpdate < 0 && foundVersion < 0) || (versionOnUpdate == 1 && foundVersion > 0)) {
                        // we're ok if versions match, or if both are negative (all missing docs are equal), or if cmd
                        // specified it must exist (versionOnUpdate==1) and it does.
                        } else {
                            throw new SolrException(ErrorCode.CONFLICT, "version conflict for " + cmd.getPrintableId() + " expected=" + versionOnUpdate + " actual=" + foundVersion);
                        }
                    }
                    long version = vinfo.getNewClock();
                    cmd.setVersion(version);
                    cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version);
                    bucket.updateHighest(version);
                } else {
                    // The leader forwarded us this update.
                    cmd.setVersion(versionOnUpdate);
                    if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (cmd.isInPlaceUpdate()) {
                        long prev = cmd.prevVersion;
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        if (lastVersion == null || Math.abs(lastVersion) < prev) {
                            // this was checked for (in waitForDependentUpdates()) before entering the synchronized block.
                            // So we shouldn't be here, unless what must've happened is:
                            // by the time synchronization block was entered, the prev update was deleted by DBQ. Since
                            // now that update is not in index, the vinfo.lookupVersion() is possibly giving us a version 
                            // from the deleted list (which might be older than the prev update!) 
                            UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate);
                            if (fetchedFromLeader instanceof DeleteUpdateCommand) {
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, and the document" + " was deleted at the leader subsequently.", idBytes.utf8ToString());
                                versionDelete((DeleteUpdateCommand) fetchedFromLeader);
                                return true;
                            } else {
                                assert fetchedFromLeader instanceof AddUpdateCommand;
                                // Newer document was fetched from the leader. Apply that document instead of this current in-place update.
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}", idBytes.utf8ToString(), fetchedFromLeader);
                                // Make this update to become a non-inplace update containing the full document obtained from the leader
                                cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc;
                                cmd.prevVersion = -1;
                                cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD));
                                assert cmd.isInPlaceUpdate() == false;
                            }
                        } else {
                            if (lastVersion != null && Math.abs(lastVersion) > prev) {
                                // this means we got a newer full doc update and in that case it makes no sense to apply the older
                                // inplace update. Drop this update
                                log.info("Update was applied on version: " + prev + ", but last version I have is: " + lastVersion + ". Dropping current update.");
                                return true;
                            } else {
                                // We're good, we should apply this update. First, update the bucket's highest.
                                if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                                    bucket.updateHighest(versionOnUpdate);
                                }
                            }
                        }
                    }
                    if (!cmd.isInPlaceUpdate()) {
                        // if we aren't the leader, then we need to check that updates were not re-ordered
                        if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                            // we're OK... this update has a version higher than anything we've seen
                            // in this bucket so far, so we know that no reordering has yet occurred.
                            bucket.updateHighest(versionOnUpdate);
                        } else {
                            // there have been updates higher than the current update.  we need to check
                            // the specific version for this id.
                            Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                            if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
                                // This update is a repeat, or was reordered.  We need to drop this update.
                                log.debug("Dropping add update due to version {}", idBytes.utf8ToString());
                                return true;
                            }
                            // also need to re-apply newer deleteByQuery commands
                            checkDeleteByQueries = true;
                        }
                    }
                    if (replicaType == Replica.Type.TLOG && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);
                    }
                }
            }
            boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
            SolrInputDocument clonedDoc = null;
            if (willDistrib && cloneRequiredOnLeader) {
                clonedDoc = cmd.solrDoc.deepCopy();
            }
            // TODO: possibly set checkDeleteByQueries as a flag on the command?
            doLocalAdd(cmd);
            if (willDistrib && cloneRequiredOnLeader) {
                cmd.solrDoc = clonedDoc;
            }
        }
    // end synchronized (bucket)
    } finally {
        vinfo.unlockForUpdate();
    }
    return false;
}

Example 2 with VersionBucket

use of org.apache.solr.update.VersionBucket in project lucene-solr by apache.

the class DistributedUpdateProcessor method versionDelete.

protected boolean versionDelete(DeleteUpdateCommand cmd) throws IOException {
    BytesRef idBytes = cmd.getIndexedId();
    if (vinfo == null || idBytes == null) {
        super.processDelete(cmd);
        return false;
    }
    // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not use a pluggable hash here)
    int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
    // at this point, there is an update we need to try and apply.
    // we may or may not be the leader.
    // Find the version
    long versionOnUpdate = cmd.getVersion();
    if (versionOnUpdate == 0) {
        String versionOnUpdateS = req.getParams().get(CommonParams.VERSION_FIELD);
        versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
    }
    long signedVersionOnUpdate = versionOnUpdate;
    // normalize to positive version
    versionOnUpdate = Math.abs(versionOnUpdate);
    boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
    boolean leaderLogic = isLeader && !isReplayOrPeersync;
    boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
    if (!leaderLogic && versionOnUpdate == 0) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader");
    }
    VersionBucket bucket = vinfo.bucket(bucketHash);
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            if (versionsStored) {
                long bucketVersion = bucket.highest;
                if (leaderLogic) {
                    if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {
                        // forwarded from a collection but we are not buffering so strip original version and apply our own
                        // see SOLR-5308
                        log.info("Removing version field from doc: " + cmd.getId());
                        versionOnUpdate = signedVersionOnUpdate = 0;
                    }
                    // leaders can also be in buffering state during "migrate" API call, see SOLR-5308
                    if (forwardedFromCollection && ulog.getState() != UpdateLog.State.ACTIVE && !isReplayOrPeersync) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        log.info("Leader logic applied but update log is buffering: " + cmd.getId());
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.delete(cmd);
                        return true;
                    }
                    if (signedVersionOnUpdate != 0) {
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        long foundVersion = lastVersion == null ? -1 : lastVersion;
                        if ((signedVersionOnUpdate == foundVersion) || (signedVersionOnUpdate < 0 && foundVersion < 0) || (signedVersionOnUpdate == 1 && foundVersion > 0)) {
                        // we're ok if versions match, or if both are negative (all missing docs are equal), or if cmd
                        // specified it must exist (versionOnUpdate==1) and it does.
                        } else {
                            throw new SolrException(ErrorCode.CONFLICT, "version conflict for " + cmd.getId() + " expected=" + signedVersionOnUpdate + " actual=" + foundVersion);
                        }
                    }
                    long version = vinfo.getNewClock();
                    cmd.setVersion(-version);
                    bucket.updateHighest(version);
                } else {
                    cmd.setVersion(-versionOnUpdate);
                    if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.delete(cmd);
                        return true;
                    }
                    // if we aren't the leader, then we need to check that updates were not re-ordered
                    if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                        // we're OK... this update has a version higher than anything we've seen
                        // in this bucket so far, so we know that no reordering has yet occured.
                        bucket.updateHighest(versionOnUpdate);
                    } else {
                        // there have been updates higher than the current update.  we need to check
                        // the specific version for this id.
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
                            // This update is a repeat, or was reordered.  We need to drop this update.
                            log.debug("Dropping delete update due to version {}", idBytes.utf8ToString());
                            return true;
                        }
                    }
                    if (replicaType == Replica.Type.TLOG && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);
                    }
                }
            }
            doLocalDelete(cmd);
            return false;
        }
    // end synchronized (bucket)
    } finally {
        vinfo.unlockForUpdate();
    }
}