Search in sources :

Example 1 with SolrApiNodeStatus

use of org.alfresco.solr.client.Node.SolrApiNodeStatus in project SearchServices by Alfresco.

the class SolrInformationServer method indexNodes.

@Override
public void indexNodes(List<Node> nodes, boolean overwrite, boolean cascade) throws IOException, AuthenticationException, JSONException {
    SolrQueryRequest request = null;
    UpdateRequestProcessor processor = null;
    try {
        request = getLocalSolrQueryRequest();
        processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
        Map<Long, Node> nodeIdsToNodes = new HashMap<>();
        EnumMap<SolrApiNodeStatus, List<Long>> nodeStatusToNodeIds = new EnumMap<SolrApiNodeStatus, List<Long>>(SolrApiNodeStatus.class);
        categorizeNodes(nodes, nodeIdsToNodes, nodeStatusToNodeIds);
        List<Long> deletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.DELETED));
        List<Long> shardDeletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED));
        List<Long> shardUpdatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED));
        List<Long> unknownNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UNKNOWN));
        List<Long> updatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UPDATED));
        if (!deletedNodeIds.isEmpty() || !shardDeletedNodeIds.isEmpty() || !shardUpdatedNodeIds.isEmpty() || !unknownNodeIds.isEmpty()) {
            // fix up any secondary paths
            List<NodeMetaData> nodeMetaDatas = new ArrayList<>();
            // For all deleted nodes, fake the node metadata
            for (Long deletedNodeId : deletedNodeIds) {
                Node node = nodeIdsToNodes.get(deletedNodeId);
                NodeMetaData nodeMetaData = createDeletedNodeMetaData(node);
                nodeMetaDatas.add(nodeMetaData);
            }
            if (!unknownNodeIds.isEmpty()) {
                NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                nmdp.setNodeIds(unknownNodeIds);
                nodeMetaDatas.addAll(repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE));
            }
            for (NodeMetaData nodeMetaData : nodeMetaDatas) {
                Node node = nodeIdsToNodes.get(nodeMetaData.getId());
                if (nodeMetaData.getTxnId() > node.getTxnId()) {
                    // it will be indexed later
                    continue;
                }
                if (nodeMetaData != null) {
                    try {
                        // Lock the node to ensure that no other trackers work with this node until this code completes.
                        if (!spinLock(nodeMetaData.getId(), 120000)) {
                            // We haven't acquired the lock in over 2 minutes. This really shouldn't happen unless something has gone wrong.
                            throw new Exception("Unable to acquire lock on nodeId:" + nodeMetaData.getId());
                        }
                        solrContentStore.removeDocFromContentStore(nodeMetaData);
                    } finally {
                        unlock(nodeMetaData.getId());
                    }
                }
            }
            if (log.isDebugEnabled()) {
                log.debug(".. deleting");
            }
            DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request);
            String query = this.cloud.getQuery(FIELD_DBID, OR, deletedNodeIds, shardDeletedNodeIds, shardUpdatedNodeIds, unknownNodeIds);
            delDocCmd.setQuery(query);
            processor.processDelete(delDocCmd);
        }
        if (!updatedNodeIds.isEmpty() || !unknownNodeIds.isEmpty() || !shardUpdatedNodeIds.isEmpty()) {
            log.info(".. updating");
            NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
            List<Long> nodeIds = new LinkedList<>();
            nodeIds.addAll(updatedNodeIds);
            nodeIds.addAll(unknownNodeIds);
            nodeIds.addAll(shardUpdatedNodeIds);
            nmdp.setNodeIds(nodeIds);
            // Fetches bulk metadata
            List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);
            NEXT_NODE: for (NodeMetaData nodeMetaData : nodeMetaDatas) {
                // System.out.println("####################### NodeMetaData:"+ nodeMetaData.getId());
                long start = System.nanoTime();
                Node node = nodeIdsToNodes.get(nodeMetaData.getId());
                long nodeId = node.getId();
                try {
                    // Lock the node to ensure that no other trackers work with this node until this code completes.
                    if (!spinLock(nodeId, 120000)) {
                        // We haven't acquired the lock in over 2 minutes. This really shouldn't happen unless something has gone wrong.
                        throw new Exception("Unable to acquire lock on nodeId:" + nodeId);
                    }
                    if (nodeMetaData.getTxnId() > node.getTxnId()) {
                        // it will be indexed later
                        continue;
                    }
                    if (nodeIdsToNodes.get(nodeMetaData.getId()).getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) {
                        if (nodeMetaData.getProperties().get(ContentModel.PROP_CASCADE_TX) != null) {
                            indexNonShardCascade(nodeMetaData);
                        }
                        continue;
                    }
                    AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                    addDocCmd.overwrite = overwrite;
                    // check index control
                    Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
                    StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_INDEXED);
                    if (pValue != null) {
                        Boolean isIndexed = Boolean.valueOf(pValue.getValue());
                        if (!isIndexed.booleanValue()) {
                            if (log.isDebugEnabled()) {
                                log.debug(".. clearing unindexed");
                            }
                            deleteNode(processor, request, node);
                            SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_UNINDEXED_NODE);
                            addDocCmd.solrDoc = doc;
                            if (recordUnindexedNodes) {
                                solrContentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
                                processor.processAdd(addDocCmd);
                            }
                            long end = System.nanoTime();
                            this.trackerStats.addNodeTime(end - start);
                            continue NEXT_NODE;
                        }
                    }
                    // Make sure any unindexed or error doc is removed.
                    if (log.isDebugEnabled()) {
                        log.debug(".. deleting node " + node.getId());
                    }
                    deleteNode(processor, request, node);
                    SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_NODE);
                    addToNewDocAndCache(nodeMetaData, doc);
                    addDocCmd.solrDoc = doc;
                    // System.out.println("###################### indexing doc:"+doc.toString());
                    processor.processAdd(addDocCmd);
                    long end = System.nanoTime();
                    this.trackerStats.addNodeTime(end - start);
                } finally {
                    // release the lock on the node so other trackers can access the node.
                    unlock(nodeId);
                }
            }
        // Ends iteration over nodeMetadatas
        }
    // Ends checking for the existence of updated or unknown node ids
    } catch (Exception e) {
        log.error("SolrInformationServer problem", e);
        // Bulk version failed, so do one at a time.
        for (Node node : nodes) {
            this.indexNode(node, true);
        }
    } finally {
        if (processor != null) {
            processor.finish();
        }
        if (request != null) {
            request.close();
        }
    }
}
Also used : StringPropertyValue(org.alfresco.solr.client.StringPropertyValue) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Node(org.alfresco.solr.client.Node) IntArrayList(com.carrotsearch.hppc.IntArrayList) ArrayList(java.util.ArrayList) SolrInputDocument(org.apache.solr.common.SolrInputDocument) SolrApiNodeStatus(org.alfresco.solr.client.Node.SolrApiNodeStatus) UpdateRequestProcessor(org.apache.solr.update.processor.UpdateRequestProcessor) NodeMetaDataParameters(org.alfresco.solr.client.NodeMetaDataParameters) SolrDocumentList(org.apache.solr.common.SolrDocumentList) IntArrayList(com.carrotsearch.hppc.IntArrayList) ArrayList(java.util.ArrayList) NamedList(org.apache.solr.common.util.NamedList) DocList(org.apache.solr.search.DocList) List(java.util.List) LinkedList(java.util.LinkedList) EnumMap(java.util.EnumMap) SolrQueryResponse(org.apache.solr.response.SolrQueryResponse) NodeMetaData(org.alfresco.solr.client.NodeMetaData) JSONException(org.json.JSONException) AuthenticationException(org.alfresco.httpclient.AuthenticationException) IOException(java.io.IOException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) LinkedList(java.util.LinkedList) LocalSolrQueryRequest(org.apache.solr.request.LocalSolrQueryRequest) SolrQueryRequest(org.apache.solr.request.SolrQueryRequest) DeleteUpdateCommand(org.apache.solr.update.DeleteUpdateCommand) AddUpdateCommand(org.apache.solr.update.AddUpdateCommand) Map(java.util.Map) SolrSimpleOrderedMap(org.alfresco.solr.adapters.SolrSimpleOrderedMap) EnumMap(java.util.EnumMap) LinkedHashMap(java.util.LinkedHashMap) SimpleOrderedMap(org.apache.solr.common.util.SimpleOrderedMap) HashMap(java.util.HashMap) ISimpleOrderedMap(org.alfresco.solr.adapters.ISimpleOrderedMap)

Aggregations

IntArrayList (com.carrotsearch.hppc.IntArrayList)1 IOException (java.io.IOException)1 UnsupportedEncodingException (java.io.UnsupportedEncodingException)1 ArrayList (java.util.ArrayList)1 EnumMap (java.util.EnumMap)1 HashMap (java.util.HashMap)1 LinkedHashMap (java.util.LinkedHashMap)1 LinkedList (java.util.LinkedList)1 List (java.util.List)1 Map (java.util.Map)1 AuthenticationException (org.alfresco.httpclient.AuthenticationException)1 ISimpleOrderedMap (org.alfresco.solr.adapters.ISimpleOrderedMap)1 SolrSimpleOrderedMap (org.alfresco.solr.adapters.SolrSimpleOrderedMap)1 Node (org.alfresco.solr.client.Node)1 SolrApiNodeStatus (org.alfresco.solr.client.Node.SolrApiNodeStatus)1 NodeMetaData (org.alfresco.solr.client.NodeMetaData)1 NodeMetaDataParameters (org.alfresco.solr.client.NodeMetaDataParameters)1 StringPropertyValue (org.alfresco.solr.client.StringPropertyValue)1 SolrDocumentList (org.apache.solr.common.SolrDocumentList)1 SolrInputDocument (org.apache.solr.common.SolrInputDocument)1