Examples with AddUpdateCommand org.apache.solr.update.AddUpdateCommand used on opensource projects

Example 76 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class ChangedSchemaMergeTest method addDoc.

private void addDoc(SolrCore core, String... fieldValues) throws IOException {
    UpdateHandler updater = core.getUpdateHandler();
    AddUpdateCommand cmd = new AddUpdateCommand(new LocalSolrQueryRequest(core, new NamedList<>()));
    cmd.solrDoc = sdoc((Object[]) fieldValues);
    updater.addDoc(cmd);
}

Example 77 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class DistributedUpdateProcessor method waitForDependentUpdates.

/**
   * This method checks the update/transaction logs and index to find out if the update ("previous update") that the current update
   * depends on (in the case that this current update is an in-place update) has already been completed. If not,
   * this method will wait for the missing update until it has arrived. If it doesn't arrive within a timeout threshold,
   * then this actively fetches from the leader.
   * 
   * @return -1 if the current in-place should be dropped, or last found version if previous update has been indexed.
   */
private long waitForDependentUpdates(AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync, VersionBucket bucket) throws IOException {
    long lastFoundVersion = 0;
    TimeOut waitTimeout = new TimeOut(5, TimeUnit.SECONDS);
    vinfo.lockForUpdate();
    try {
        synchronized (bucket) {
            Long lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());
            lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;
            if (Math.abs(lastFoundVersion) < cmd.prevVersion) {
                log.debug("Re-ordered inplace update. version={}, prevVersion={}, lastVersion={}, replayOrPeerSync={}, id={}", (cmd.getVersion() == 0 ? versionOnUpdate : cmd.getVersion()), cmd.prevVersion, lastFoundVersion, isReplayOrPeersync, cmd.getPrintableId());
            }
            while (Math.abs(lastFoundVersion) < cmd.prevVersion && !waitTimeout.hasTimedOut()) {
                try {
                    long timeLeft = waitTimeout.timeLeft(TimeUnit.MILLISECONDS);
                    if (timeLeft > 0) {
                        // wait(0) waits forever until notified, but we don't want that.
                        bucket.wait(timeLeft);
                    }
                } catch (InterruptedException ie) {
                    throw new RuntimeException(ie);
                }
                lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());
                lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;
            }
        }
    } finally {
        vinfo.unlockForUpdate();
    }
    if (Math.abs(lastFoundVersion) > cmd.prevVersion) {
        // we can drop the current update.
        if (log.isDebugEnabled()) {
            log.debug("Update was applied on version: {}, but last version I have is: {}" + ". Current update should be dropped. id={}", cmd.prevVersion, lastFoundVersion, cmd.getPrintableId());
        }
        return -1;
    } else if (Math.abs(lastFoundVersion) == cmd.prevVersion) {
        assert 0 < lastFoundVersion : "prevVersion " + cmd.prevVersion + " found but is a delete!";
        if (log.isDebugEnabled()) {
            log.debug("Dependent update found. id={}", cmd.getPrintableId());
        }
        return lastFoundVersion;
    }
    // We have waited enough, but dependent update didn't arrive. Its time to actively fetch it from leader
    log.info("Missing update, on which current in-place update depends on, hasn't arrived. id={}, looking for version={}, last found version={}", cmd.getPrintableId(), cmd.prevVersion, lastFoundVersion);
    UpdateCommand missingUpdate = fetchFullUpdateFromLeader(cmd, versionOnUpdate);
    if (missingUpdate instanceof DeleteUpdateCommand) {
        log.info("Tried to fetch document {} from the leader, but the leader says document has been deleted. " + "Deleting the document here and skipping this update: Last found version: {}, was looking for: {}", cmd.getPrintableId(), lastFoundVersion, cmd.prevVersion);
        versionDelete((DeleteUpdateCommand) missingUpdate);
        return -1;
    } else {
        assert missingUpdate instanceof AddUpdateCommand;
        log.debug("Fetched the document: {}", ((AddUpdateCommand) missingUpdate).getSolrInputDocument());
        versionAdd((AddUpdateCommand) missingUpdate);
        log.info("Added the fetched document, id=" + ((AddUpdateCommand) missingUpdate).getPrintableId() + ", version=" + missingUpdate.getVersion());
    }
    return missingUpdate.getVersion();
}

Example 78 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class FieldNameMutatingUpdateProcessorFactory method getInstance.

@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
    return new UpdateRequestProcessor(next) {

        @Override
        public void processAdd(AddUpdateCommand cmd) throws IOException {
            final SolrInputDocument doc = cmd.getSolrInputDocument();
            final Collection<String> fieldNames = new ArrayList<>(doc.getFieldNames());
            for (final String fname : fieldNames) {
                Matcher matcher = pattern.matcher(fname);
                if (matcher.find()) {
                    String newFieldName = matcher.replaceAll(replacement);
                    if (!newFieldName.equals(fname)) {
                        SolrInputField old = doc.remove(fname);
                        old.setName(newFieldName);
                        doc.put(newFieldName, old);
                    }
                }
            }
            super.processAdd(cmd);
        }

        @Override
        public void processDelete(DeleteUpdateCommand cmd) throws IOException {
            super.processDelete(cmd);
        }
    };
}

Example 79 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class TemplateUpdateProcessorFactory method process.

@Override
protected void process(AddUpdateCommand cmd, SolrQueryRequest req, SolrQueryResponse rsp) {
    String[] vals = getParams("field");
    SolrInputDocument doc = cmd.getSolrInputDocument();
    if (vals != null && vals.length > 0) {
        for (String val : vals) {
            if (val == null || val.isEmpty())
                continue;
            int idx = val.indexOf(':');
            if (idx == -1)
                throw new RuntimeException("'field' must be of the format <field-name>:<the-template-string>");
            String fName = val.substring(0, idx);
            String template = val.substring(idx + 1);
            doc.addField(fName, replaceTokens(template, templateCache, s -> {
                Object v = doc.getFieldValue(s);
                return v == null ? "" : v;
            }));
        }
    }
}

Example 80 with AddUpdateCommand

use of org.apache.solr.update.AddUpdateCommand in project lucene-solr by apache.

the class DistributedUpdateProcessor method fetchFullUpdateFromLeader.

/**
   * This method is used when an update on which a particular in-place update has been lost for some reason. This method
   * sends a request to the shard leader to fetch the latest full document as seen on the leader.
   * @return AddUpdateCommand containing latest full doc at shard leader for the given id, or null if not found.
   */
private UpdateCommand fetchFullUpdateFromLeader(AddUpdateCommand inplaceAdd, long versionOnUpdate) throws IOException {
    String id = inplaceAdd.getPrintableId();
    UpdateShardHandler updateShardHandler = inplaceAdd.getReq().getCore().getCoreContainer().getUpdateShardHandler();
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(DISTRIB, false);
    params.set("getInputDocument", id);
    params.set("onlyIfActive", true);
    SolrRequest<SimpleSolrResponse> ur = new GenericSolrRequest(METHOD.GET, "/get", params);
    String leaderUrl = req.getParams().get(DISTRIB_FROM);
    if (leaderUrl == null) {
        // leader for the update.
        if (zkController == null) {
            // we should be in cloud mode, but wtf? could be a unit test
            throw new SolrException(ErrorCode.SERVER_ERROR, "Can't find document with id=" + id + ", but fetching from leader " + "failed since we're not in cloud mode.");
        }
        Replica leader;
        try {
            leader = zkController.getZkStateReader().getLeaderRetry(cloudDesc.getCollectionName(), cloudDesc.getShardId());
        } catch (InterruptedException e) {
            throw new SolrException(ErrorCode.SERVER_ERROR, "Exception during fetching from leader.", e);
        }
        leaderUrl = leader.getCoreUrl();
    }
    HttpSolrClient hsc = new HttpSolrClient.Builder(leaderUrl).withHttpClient(updateShardHandler.getHttpClient()).build();
    NamedList rsp = null;
    try {
        rsp = hsc.request(ur);
    } catch (SolrServerException e) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "Error during fetching [" + id + "] from leader (" + leaderUrl + "): ", e);
    } finally {
        hsc.close();
    }
    Object inputDocObj = rsp.get("inputDocument");
    Long version = (Long) rsp.get("version");
    SolrInputDocument leaderDoc = (SolrInputDocument) inputDocObj;
    if (leaderDoc == null) {
        // this doc was not found (deleted) on the leader. Lets delete it here as well.
        DeleteUpdateCommand del = new DeleteUpdateCommand(inplaceAdd.getReq());
        del.setIndexedId(inplaceAdd.getIndexedId());
        del.setId(inplaceAdd.getIndexedId().utf8ToString());
        del.setVersion((version == null || version == 0) ? -versionOnUpdate : version);
        return del;
    }
    AddUpdateCommand cmd = new AddUpdateCommand(req);
    cmd.solrDoc = leaderDoc;
    cmd.setVersion((long) leaderDoc.getFieldValue(CommonParams.VERSION_FIELD));
    return cmd;
}