Examples with AddUpdateCommand org.apache.solr.update.AddUpdateCommand used on opensource projects

Example 36 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class DistributedUpdateProcessor method isLeader.

// internal helper method to tell if we are the leader for an add or deleteById update
boolean isLeader(UpdateCommand cmd) {
    updateCommand = cmd;
    if (zkEnabled) {
        zkCheck();
        if (cmd instanceof AddUpdateCommand) {
            AddUpdateCommand acmd = (AddUpdateCommand) cmd;
            nodes = setupRequest(acmd.getHashableId(), acmd.getSolrInputDocument());
        } else if (cmd instanceof DeleteUpdateCommand) {
            DeleteUpdateCommand dcmd = (DeleteUpdateCommand) cmd;
            nodes = setupRequest(dcmd.getId(), null);
        }
    } else {
        isLeader = getNonZkLeaderAssumption(req);
    }
    return isLeader;
}

Example 37 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class DistributedUpdateProcessor method versionAdd.

/**
   * @return whether or not to drop this cmd
   * @throws IOException If there is a low-level I/O error.
   */
protected boolean versionAdd(AddUpdateCommand cmd) throws IOException {
    BytesRef idBytes = cmd.getIndexedId();
    if (idBytes == null) {
        super.processAdd(cmd);
        return false;
    }
    if (vinfo == null) {
        if (AtomicUpdateDocumentMerger.isAtomicUpdate(cmd)) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Atomic document updates are not supported unless <updateLog/> is configured");
        } else {
            super.processAdd(cmd);
            return false;
        }
    }
    // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not use a pluggable hash here)
    int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);
    // at this point, there is an update we need to try and apply.
    // we may or may not be the leader.
    // Find any existing version in the document
    // TODO: don't reuse update commands any more!
    long versionOnUpdate = cmd.getVersion();
    if (versionOnUpdate == 0) {
        SolrInputField versionField = cmd.getSolrInputDocument().getField(CommonParams.VERSION_FIELD);
        if (versionField != null) {
            Object o = versionField.getValue();
            versionOnUpdate = o instanceof Number ? ((Number) o).longValue() : Long.parseLong(o.toString());
        } else {
            // Find the version
            String versionOnUpdateS = req.getParams().get(CommonParams.VERSION_FIELD);
            versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
        }
    }
    boolean isReplayOrPeersync = (cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
    boolean leaderLogic = isLeader && !isReplayOrPeersync;
    boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;
    VersionBucket bucket = vinfo.bucket(bucketHash);
    long dependentVersionFound = -1;
    // this update depends), before entering the synchronized block
    if (!leaderLogic && cmd.isInPlaceUpdate()) {
        dependentVersionFound = waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync, bucket);
        if (dependentVersionFound == -1) {
            // it means the document has been deleted by now at the leader. drop this update
            return true;
        }
    }
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            //just in case anyone is waiting let them know that we have a new update
            bucket.notifyAll();
            // we obtain the version when synchronized and then do the add so we can ensure that
            // if version1 < version2 then version1 is actually added before version2.
            // even if we don't store the version field, synchronizing on the bucket
            // will enable us to know what version happened first, and thus enable
            // realtime-get to work reliably.
            // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?
            // there may be other reasons in the future for a version on the commands
            boolean checkDeleteByQueries = false;
            if (versionsStored) {
                long bucketVersion = bucket.highest;
                if (leaderLogic) {
                    if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {
                        // forwarded from a collection but we are not buffering so strip original version and apply our own
                        // see SOLR-5308
                        log.info("Removing version field from doc: " + cmd.getPrintableId());
                        cmd.solrDoc.remove(CommonParams.VERSION_FIELD);
                        versionOnUpdate = 0;
                    }
                    boolean updated = getUpdatedDocument(cmd, versionOnUpdate);
                    // leaders can also be in buffering state during "migrate" API call, see SOLR-5308
                    if (forwardedFromCollection && ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        log.info("Leader logic applied but update log is buffering: " + cmd.getPrintableId());
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (versionOnUpdate != 0) {
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        long foundVersion = lastVersion == null ? -1 : lastVersion;
                        if (versionOnUpdate == foundVersion || (versionOnUpdate < 0 && foundVersion < 0) || (versionOnUpdate == 1 && foundVersion > 0)) {
                        // we're ok if versions match, or if both are negative (all missing docs are equal), or if cmd
                        // specified it must exist (versionOnUpdate==1) and it does.
                        } else {
                            throw new SolrException(ErrorCode.CONFLICT, "version conflict for " + cmd.getPrintableId() + " expected=" + versionOnUpdate + " actual=" + foundVersion);
                        }
                    }
                    long version = vinfo.getNewClock();
                    cmd.setVersion(version);
                    cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version);
                    bucket.updateHighest(version);
                } else {
                    // The leader forwarded us this update.
                    cmd.setVersion(versionOnUpdate);
                    if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
                        // we're not in an active state, and this update isn't from a replay, so buffer it.
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
                        ulog.add(cmd);
                        return true;
                    }
                    if (cmd.isInPlaceUpdate()) {
                        long prev = cmd.prevVersion;
                        Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                        if (lastVersion == null || Math.abs(lastVersion) < prev) {
                            // this was checked for (in waitForDependentUpdates()) before entering the synchronized block.
                            // So we shouldn't be here, unless what must've happened is:
                            // by the time synchronization block was entered, the prev update was deleted by DBQ. Since
                            // now that update is not in index, the vinfo.lookupVersion() is possibly giving us a version 
                            // from the deleted list (which might be older than the prev update!) 
                            UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate);
                            if (fetchedFromLeader instanceof DeleteUpdateCommand) {
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, and the document" + " was deleted at the leader subsequently.", idBytes.utf8ToString());
                                versionDelete((DeleteUpdateCommand) fetchedFromLeader);
                                return true;
                            } else {
                                assert fetchedFromLeader instanceof AddUpdateCommand;
                                // Newer document was fetched from the leader. Apply that document instead of this current in-place update.
                                log.info("In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}", idBytes.utf8ToString(), fetchedFromLeader);
                                // Make this update to become a non-inplace update containing the full document obtained from the leader
                                cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc;
                                cmd.prevVersion = -1;
                                cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD));
                                assert cmd.isInPlaceUpdate() == false;
                            }
                        } else {
                            if (lastVersion != null && Math.abs(lastVersion) > prev) {
                                // this means we got a newer full doc update and in that case it makes no sense to apply the older
                                // inplace update. Drop this update
                                log.info("Update was applied on version: " + prev + ", but last version I have is: " + lastVersion + ". Dropping current update.");
                                return true;
                            } else {
                                // We're good, we should apply this update. First, update the bucket's highest.
                                if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                                    bucket.updateHighest(versionOnUpdate);
                                }
                            }
                        }
                    }
                    if (!cmd.isInPlaceUpdate()) {
                        // if we aren't the leader, then we need to check that updates were not re-ordered
                        if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                            // we're OK... this update has a version higher than anything we've seen
                            // in this bucket so far, so we know that no reordering has yet occurred.
                            bucket.updateHighest(versionOnUpdate);
                        } else {
                            // there have been updates higher than the current update.  we need to check
                            // the specific version for this id.
                            Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());
                            if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {
                                // This update is a repeat, or was reordered.  We need to drop this update.
                                log.debug("Dropping add update due to version {}", idBytes.utf8ToString());
                                return true;
                            }
                            // also need to re-apply newer deleteByQuery commands
                            checkDeleteByQueries = true;
                        }
                    }
                    if (replicaType == Replica.Type.TLOG && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
                        cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);
                    }
                }
            }
            boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
            SolrInputDocument clonedDoc = null;
            if (willDistrib && cloneRequiredOnLeader) {
                clonedDoc = cmd.solrDoc.deepCopy();
            }
            // TODO: possibly set checkDeleteByQueries as a flag on the command?
            doLocalAdd(cmd);
            if (willDistrib && cloneRequiredOnLeader) {
                cmd.solrDoc = clonedDoc;
            }
        }
    // end synchronized (bucket)
    } finally {
        vinfo.unlockForUpdate();
    }
    return false;
}

Example 38 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class JavabinLoaderTest method doTestLastDocInBatchFlag.

protected void doTestLastDocInBatchFlag(int numDocsInBatch) throws Exception {
    List<SolrInputDocument> batch = new ArrayList<>(numDocsInBatch);
    for (int d = 0; d < numDocsInBatch; d++) {
        SolrInputDocument doc = new SolrInputDocument();
        doc.setField("id", String.valueOf(d));
        batch.add(doc);
    }
    UpdateRequest updateRequest = new UpdateRequest();
    if (batch.size() > 1) {
        updateRequest.add(batch);
    } else {
        updateRequest.add(batch.get(0));
    }
    // client-side SolrJ would do this ...
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    (new JavaBinUpdateRequestCodec()).marshal(updateRequest, os);
    // need to override the processAdd method b/c JavabinLoader calls
    // clear on the addCmd after it is passed on to the handler ... a simple clone will suffice for this test
    BufferingRequestProcessor mockUpdateProcessor = new BufferingRequestProcessor(null) {

        @Override
        public void processAdd(AddUpdateCommand cmd) throws IOException {
            addCommands.add((AddUpdateCommand) cmd.clone());
        }
    };
    SolrQueryRequest req = req();
    (new JavabinLoader()).load(req, new SolrQueryResponse(), new ContentStreamBase.ByteArrayStream(os.toByteArray(), "test"), mockUpdateProcessor);
    req.close();
    assertTrue(mockUpdateProcessor.addCommands.size() == numDocsInBatch);
    for (int i = 0; i < numDocsInBatch - 1; i++) // not last doc in batch
    assertFalse(mockUpdateProcessor.addCommands.get(i).isLastDocInBatch);
    // last doc should have the flag set
    assertTrue(mockUpdateProcessor.addCommands.get(batch.size() - 1).isLastDocInBatch);
}

Example 39 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class SkipExistingDocumentsProcessorFactoryTest method createAtomicUpdateCmd.

private AddUpdateCommand createAtomicUpdateCmd(SolrQueryRequest req) {
    AddUpdateCommand cmd = new AddUpdateCommand(req);
    cmd.setIndexedId(docId);
    cmd.solrDoc = new SolrInputDocument();
    cmd.solrDoc.addField("last_name", ImmutableMap.of("set", "Smith"));
    assertTrue(AtomicUpdateDocumentMerger.isAtomicUpdate(cmd));
    return cmd;
}

Example 40 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class URLClassifyProcessorTest method testProcessor.

@Test
public void testProcessor() throws IOException {
    AddUpdateCommand addCommand = new AddUpdateCommand(null);
    SolrInputDocument document = new SolrInputDocument();
    document.addField("id", "test");
    document.addField("url", "http://www.example.com");
    addCommand.solrDoc = document;
    classifyProcessor.processAdd(addCommand);
}