Examples with UpdateRequestProcessor org.apache.solr.update.processor.UpdateRequestProcessor used on opensource projects

Example 21 with UpdateRequestProcessor

use of org.apache.solr.update.processor.UpdateRequestProcessor in project lucene-solr by apache.

the class PeerSync method handleUpdates.

private boolean handleUpdates(ShardResponse srsp) {
    // we retrieved the last N updates from the replica
    List<Object> updates = (List<Object>) srsp.getSolrResponse().getResponse().get("updates");
    SyncShardRequest sreq = (SyncShardRequest) srsp.getShardRequest();
    if (updates.size() < sreq.totalRequestedUpdates) {
        log.error(msg() + " Requested " + sreq.totalRequestedUpdates + " updates from " + sreq.shards[0] + " but retrieved " + updates.size());
        return false;
    }
    // overwrite fingerprint we saved in 'handleVersions()'   
    Object fingerprint = srsp.getSolrResponse().getResponse().get("fingerprint");
    if (fingerprint != null) {
        sreq.fingerprint = IndexFingerprint.fromObject(fingerprint);
    }
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(DISTRIB_UPDATE_PARAM, FROMLEADER.toString());
    // debugging
    params.set("peersync", true);
    SolrQueryRequest req = new LocalSolrQueryRequest(uhandler.core, params);
    SolrQueryResponse rsp = new SolrQueryResponse();
    UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessingChain(null);
    UpdateRequestProcessor proc = processorChain.createProcessor(req, rsp);
    Collections.sort(updates, updateRecordComparator);
    Object o = null;
    long lastVersion = 0;
    try {
        // Apply oldest updates first
        for (Object obj : updates) {
            // should currently be a List<Oper,Ver,Doc/Id>
            o = obj;
            List<Object> entry = (List<Object>) o;
            if (debug) {
                log.debug(msg() + "raw update record " + o);
            }
            int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
            long version = (Long) entry.get(1);
            if (version == lastVersion && version != 0)
                continue;
            lastVersion = version;
            switch(oper) {
                case UpdateLog.ADD:
                    {
                        // byte[] idBytes = (byte[]) entry.get(2);
                        SolrInputDocument sdoc = (SolrInputDocument) entry.get(entry.size() - 1);
                        AddUpdateCommand cmd = new AddUpdateCommand(req);
                        // cmd.setIndexedId(new BytesRef(idBytes));
                        cmd.solrDoc = sdoc;
                        cmd.setVersion(version);
                        cmd.setFlags(UpdateCommand.PEER_SYNC | UpdateCommand.IGNORE_AUTOCOMMIT);
                        if (debug) {
                            log.debug(msg() + "add " + cmd + " id " + sdoc.getField(ID));
                        }
                        proc.processAdd(cmd);
                        break;
                    }
                case UpdateLog.DELETE:
                    {
                        byte[] idBytes = (byte[]) entry.get(2);
                        DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
                        cmd.setIndexedId(new BytesRef(idBytes));
                        cmd.setVersion(version);
                        cmd.setFlags(UpdateCommand.PEER_SYNC | UpdateCommand.IGNORE_AUTOCOMMIT);
                        if (debug) {
                            log.debug(msg() + "delete " + cmd + " " + new BytesRef(idBytes).utf8ToString());
                        }
                        proc.processDelete(cmd);
                        break;
                    }
                case UpdateLog.DELETE_BY_QUERY:
                    {
                        String query = (String) entry.get(2);
                        DeleteUpdateCommand cmd = new DeleteUpdateCommand(req);
                        cmd.query = query;
                        cmd.setVersion(version);
                        cmd.setFlags(UpdateCommand.PEER_SYNC | UpdateCommand.IGNORE_AUTOCOMMIT);
                        if (debug) {
                            log.debug(msg() + "deleteByQuery " + cmd);
                        }
                        proc.processDelete(cmd);
                        break;
                    }
                case UpdateLog.UPDATE_INPLACE:
                    {
                        AddUpdateCommand cmd = UpdateLog.convertTlogEntryToAddUpdateCommand(req, entry, oper, version);
                        cmd.setFlags(UpdateCommand.PEER_SYNC | UpdateCommand.IGNORE_AUTOCOMMIT);
                        if (debug) {
                            log.debug(msg() + "inplace update " + cmd + " prevVersion=" + cmd.prevVersion + ", doc=" + cmd.solrDoc);
                        }
                        proc.processAdd(cmd);
                        break;
                    }
                default:
                    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
            }
        }
    } catch (IOException e) {
        // TODO: should this be handled separately as a problem with us?
        // I guess it probably already will by causing replication to be kicked off.
        sreq.updateException = e;
        log.error(msg() + "Error applying updates from " + sreq.shards + " ,update=" + o, e);
        return false;
    } catch (Exception e) {
        sreq.updateException = e;
        log.error(msg() + "Error applying updates from " + sreq.shards + " ,update=" + o, e);
        return false;
    } finally {
        try {
            proc.finish();
        } catch (Exception e) {
            sreq.updateException = e;
            log.error(msg() + "Error applying updates from " + sreq.shards + " ,finish()", e);
            return false;
        } finally {
            IOUtils.closeQuietly(proc);
        }
    }
    return compareFingerprint(sreq);
}

Example 22 with UpdateRequestProcessor

use of org.apache.solr.update.processor.UpdateRequestProcessor in project SearchServices by Alfresco.

the class SolrInformationServer method indexNode.

@Override
public void indexNode(Node node, boolean overwrite) throws IOException, AuthenticationException, JSONException {
    SolrQueryRequest request = null;
    UpdateRequestProcessor processor = null;
    try {
        request = getLocalSolrQueryRequest();
        processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
        long start = System.nanoTime();
        if ((node.getStatus() == SolrApiNodeStatus.DELETED) || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) {
            // fix up any secondary paths
            NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
            nmdp.setFromNodeId(node.getId());
            nmdp.setToNodeId(node.getId());
            List<NodeMetaData> nodeMetaDatas;
            if ((node.getStatus() == SolrApiNodeStatus.DELETED) || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED) || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) {
                // Fake the empty node metadata for this parent deleted node
                NodeMetaData nodeMetaData = createDeletedNodeMetaData(node);
                nodeMetaDatas = Collections.singletonList(nodeMetaData);
            } else {
                nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);
            }
            NodeMetaData nodeMetaData = null;
            if (!nodeMetaDatas.isEmpty()) {
                nodeMetaData = nodeMetaDatas.get(0);
                if (!(nodeMetaData.getTxnId() > node.getTxnId())) {
                    if (node.getStatus() == SolrApiNodeStatus.DELETED) {
                        try {
                            // Lock the node to ensure that no other trackers work with this node until this code completes.
                            if (!spinLock(nodeMetaData.getId(), 120000)) {
                                // We haven't acquired the lock in over 2 minutes. This really shouldn't happen unless something has gone wrong.
                                throw new Exception("Unable to acquire lock on nodeId:" + nodeMetaData.getId());
                            }
                            solrContentStore.removeDocFromContentStore(nodeMetaData);
                        } finally {
                            unlock(nodeMetaData.getId());
                        }
                    }
                }
            // else, the node has moved on to a later transaction, and it will be indexed later
            }
            if (log.isDebugEnabled()) {
                log.debug(".. deleting");
            }
            deleteNode(processor, request, node);
        }
        if ((node.getStatus() == SolrApiNodeStatus.UPDATED) || (node.getStatus() == SolrApiNodeStatus.UNKNOWN) || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) {
            log.info(".. updating");
            long nodeId = node.getId();
            try {
                if (!spinLock(nodeId, 120000)) {
                    // We haven't acquired the lock in over 2 minutes. This really shouldn't happen unless something has gone wrong.
                    throw new Exception("Unable to acquire lock on nodeId:" + nodeId);
                }
                NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                nmdp.setFromNodeId(node.getId());
                nmdp.setToNodeId(node.getId());
                List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);
                AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                addDocCmd.overwrite = overwrite;
                if (!nodeMetaDatas.isEmpty()) {
                    NodeMetaData nodeMetaData = nodeMetaDatas.get(0);
                    if (!(nodeMetaData.getTxnId() > node.getTxnId())) {
                    /*
                        if (mayHaveChildren(nodeMetaData))
                        {
                            cascadeUpdate(nodeMetaData, overwrite, request, processor);
                        }
            				 */
                    }
                    if (node.getTxnId() == Long.MAX_VALUE) {
                        // This is a re-index. We need to clear the txnId from the pr
                        this.cleanContentCache.remove(nodeMetaData.getTxnId());
                    }
                    if ((node.getStatus() == SolrApiNodeStatus.UPDATED) || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) {
                        // check index control
                        Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
                        StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_INDEXED);
                        if (pValue != null) {
                            Boolean isIndexed = Boolean.valueOf(pValue.getValue());
                            if (!isIndexed.booleanValue()) {
                                if (log.isDebugEnabled()) {
                                    log.debug(".. clearing unindexed");
                                }
                                deleteNode(processor, request, node);
                                SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_UNINDEXED_NODE);
                                solrContentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
                                addDocCmd.solrDoc = doc;
                                processor.processAdd(addDocCmd);
                                long end = System.nanoTime();
                                this.trackerStats.addNodeTime(end - start);
                                return;
                            }
                        }
                        // Make sure any unindexed or error doc is removed.
                        if (log.isDebugEnabled()) {
                            log.debug(".. deleting node " + node.getId());
                        }
                        deleteNode(processor, request, node);
                        SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_NODE);
                        addToNewDocAndCache(nodeMetaData, doc);
                        addDocCmd.solrDoc = doc;
                        processor.processAdd(addDocCmd);
                    }
                }
            // Ends checking for a nodeMetaData
            } finally {
                unlock(nodeId);
            }
        }
        // Ends checking for updated or unknown node status
        long end = System.nanoTime();
        this.trackerStats.addNodeTime(end - start);
    } catch (Exception e) {
        log.warn("Node index failed and skipped for " + node.getId() + " in Tx " + node.getTxnId(), e);
        if (processor == null) {
            if (request == null) {
                request = getLocalSolrQueryRequest();
            }
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
        }
        if (log.isDebugEnabled()) {
            log.debug(".. deleting on exception");
        }
        deleteNode(processor, request, node);
        AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
        addDocCmd.overwrite = overwrite;
        SolrInputDocument doc = new SolrInputDocument();
        doc.addField(FIELD_SOLR4_ID, PREFIX_ERROR + node.getId());
        doc.addField(FIELD_VERSION, "0");
        doc.addField(FIELD_DBID, node.getId());
        doc.addField(FIELD_INTXID, node.getTxnId());
        doc.addField(FIELD_EXCEPTION_MESSAGE, e.getMessage());
        doc.addField(FIELD_DOC_TYPE, DOC_TYPE_ERROR_NODE);
        StringWriter stringWriter = new StringWriter(4096);
        PrintWriter printWriter = new PrintWriter(stringWriter, true);
        try {
            e.printStackTrace(printWriter);
            String stack = stringWriter.toString();
            doc.addField(FIELD_EXCEPTION_STACK, stack.length() < 32766 ? stack : stack.substring(0, 32765));
        } finally {
            printWriter.close();
        }
        addDocCmd.solrDoc = doc;
        processor.processAdd(addDocCmd);
    } finally {
        if (processor != null) {
            processor.finish();
        }
        if (request != null) {
            request.close();
        }
    }
}

Example 23 with UpdateRequestProcessor

use of org.apache.solr.update.processor.UpdateRequestProcessor in project SearchServices by Alfresco.

the class SolrInformationServer method indexAclTransaction.

@Override
public void indexAclTransaction(AclChangeSet changeSet, boolean overwrite) throws IOException {
    canUpdate();
    SolrQueryRequest request = null;
    UpdateRequestProcessor processor = null;
    try {
        request = getLocalSolrQueryRequest();
        processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
        AddUpdateCommand cmd = new AddUpdateCommand(request);
        cmd.overwrite = overwrite;
        SolrInputDocument input = new SolrInputDocument();
        input.addField(FIELD_SOLR4_ID, AlfrescoSolrDataModel.getAclChangeSetDocumentId(changeSet.getId()));
        input.addField(FIELD_VERSION, "0");
        input.addField(FIELD_ACLTXID, changeSet.getId());
        input.addField(FIELD_INACLTXID, changeSet.getId());
        input.addField(FIELD_ACLTXCOMMITTIME, changeSet.getCommitTimeMs());
        input.addField(FIELD_DOC_TYPE, DOC_TYPE_ACL_TX);
        cmd.solrDoc = input;
        processor.processAdd(cmd);
        putAclTransactionState(processor, request, changeSet);
    } finally {
        if (processor != null) {
            processor.finish();
        }
        if (request != null) {
            request.close();
        }
    }
}

Example 24 with UpdateRequestProcessor

use of org.apache.solr.update.processor.UpdateRequestProcessor in project SearchServices by Alfresco.

the class SolrInformationServer method commit.

@Override
public void commit() throws IOException {
    // avoid multiple commits and warming searchers
    commitAndRollbackLock.writeLock().lock();
    try {
        canUpdate();
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
            processor.processCommit(new CommitUpdateCommand(request, false));
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }
    } finally {
        commitAndRollbackLock.writeLock().unlock();
    }
}

Example 25 with UpdateRequestProcessor

use of org.apache.solr.update.processor.UpdateRequestProcessor in project SearchServices by Alfresco.

the class SolrInformationServer method indexNonShardCascade.

private void indexNonShardCascade(NodeMetaData nodeMetaData) throws IOException {
    canUpdate();
    SolrQueryRequest request = null;
    UpdateRequestProcessor processor = null;
    try {
        request = getLocalSolrQueryRequest();
        processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
        StringPropertyValue stringPropertyValue = (StringPropertyValue) nodeMetaData.getProperties().get(ContentModel.PROP_CASCADE_TX);
        List<FieldInstance> fieldInstances = AlfrescoSolrDataModel.getInstance().getIndexedFieldNamesForProperty(ContentModel.PROP_CASCADE_TX).getFields();
        FieldInstance fieldInstance = fieldInstances.get(0);
        AddUpdateCommand cmd = new AddUpdateCommand(request);
        SolrInputDocument input = new SolrInputDocument();
        input.addField(FIELD_SOLR4_ID, AlfrescoSolrDataModel.getNodeDocumentId(nodeMetaData.getTenantDomain(), nodeMetaData.getAclId(), nodeMetaData.getId()));
        input.addField(FIELD_VERSION, 0);
        input.addField(fieldInstance.getField(), stringPropertyValue.toString());
        cmd.solrDoc = input;
        processor.processAdd(cmd);
    } finally {
        if (processor != null) {
            processor.finish();
        }
        if (request != null) {
            request.close();
        }
    }
}