use of org.apache.solr.update.processor.UpdateRequestProcessor in project SearchServices by Alfresco.
the class SolrInformationServer method indexNodes.
@Override
public void indexNodes(List<Node> nodes, boolean overwrite, boolean cascade) throws IOException, AuthenticationException, JSONException {
SolrQueryRequest request = null;
UpdateRequestProcessor processor = null;
try {
request = getLocalSolrQueryRequest();
processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
Map<Long, Node> nodeIdsToNodes = new HashMap<>();
EnumMap<SolrApiNodeStatus, List<Long>> nodeStatusToNodeIds = new EnumMap<SolrApiNodeStatus, List<Long>>(SolrApiNodeStatus.class);
categorizeNodes(nodes, nodeIdsToNodes, nodeStatusToNodeIds);
List<Long> deletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.DELETED));
List<Long> shardDeletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED));
List<Long> shardUpdatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED));
List<Long> unknownNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UNKNOWN));
List<Long> updatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UPDATED));
if (!deletedNodeIds.isEmpty() || !shardDeletedNodeIds.isEmpty() || !shardUpdatedNodeIds.isEmpty() || !unknownNodeIds.isEmpty()) {
// fix up any secondary paths
List<NodeMetaData> nodeMetaDatas = new ArrayList<>();
// For all deleted nodes, fake the node metadata
for (Long deletedNodeId : deletedNodeIds) {
Node node = nodeIdsToNodes.get(deletedNodeId);
NodeMetaData nodeMetaData = createDeletedNodeMetaData(node);
nodeMetaDatas.add(nodeMetaData);
}
if (!unknownNodeIds.isEmpty()) {
NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
nmdp.setNodeIds(unknownNodeIds);
nodeMetaDatas.addAll(repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE));
}
for (NodeMetaData nodeMetaData : nodeMetaDatas) {
Node node = nodeIdsToNodes.get(nodeMetaData.getId());
if (nodeMetaData.getTxnId() > node.getTxnId()) {
// it will be indexed later
continue;
}
if (nodeMetaData != null) {
try {
// Lock the node to ensure that no other trackers work with this node until this code completes.
if (!spinLock(nodeMetaData.getId(), 120000)) {
// We haven't acquired the lock in over 2 minutes. This really shouldn't happen unless something has gone wrong.
throw new Exception("Unable to acquire lock on nodeId:" + nodeMetaData.getId());
}
solrContentStore.removeDocFromContentStore(nodeMetaData);
} finally {
unlock(nodeMetaData.getId());
}
}
}
if (log.isDebugEnabled()) {
log.debug(".. deleting");
}
DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request);
String query = this.cloud.getQuery(FIELD_DBID, OR, deletedNodeIds, shardDeletedNodeIds, shardUpdatedNodeIds, unknownNodeIds);
delDocCmd.setQuery(query);
processor.processDelete(delDocCmd);
}
if (!updatedNodeIds.isEmpty() || !unknownNodeIds.isEmpty() || !shardUpdatedNodeIds.isEmpty()) {
log.info(".. updating");
NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
List<Long> nodeIds = new LinkedList<>();
nodeIds.addAll(updatedNodeIds);
nodeIds.addAll(unknownNodeIds);
nodeIds.addAll(shardUpdatedNodeIds);
nmdp.setNodeIds(nodeIds);
// Fetches bulk metadata
List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);
NEXT_NODE: for (NodeMetaData nodeMetaData : nodeMetaDatas) {
// System.out.println("####################### NodeMetaData:"+ nodeMetaData.getId());
long start = System.nanoTime();
Node node = nodeIdsToNodes.get(nodeMetaData.getId());
long nodeId = node.getId();
try {
// Lock the node to ensure that no other trackers work with this node until this code completes.
if (!spinLock(nodeId, 120000)) {
// We haven't acquired the lock in over 2 minutes. This really shouldn't happen unless something has gone wrong.
throw new Exception("Unable to acquire lock on nodeId:" + nodeId);
}
if (nodeMetaData.getTxnId() > node.getTxnId()) {
// it will be indexed later
continue;
}
if (nodeIdsToNodes.get(nodeMetaData.getId()).getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) {
if (nodeMetaData.getProperties().get(ContentModel.PROP_CASCADE_TX) != null) {
indexNonShardCascade(nodeMetaData);
}
continue;
}
AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
addDocCmd.overwrite = overwrite;
// check index control
Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_INDEXED);
if (pValue != null) {
Boolean isIndexed = Boolean.valueOf(pValue.getValue());
if (!isIndexed.booleanValue()) {
if (log.isDebugEnabled()) {
log.debug(".. clearing unindexed");
}
deleteNode(processor, request, node);
SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_UNINDEXED_NODE);
addDocCmd.solrDoc = doc;
if (recordUnindexedNodes) {
solrContentStore.storeDocOnSolrContentStore(nodeMetaData, doc);
processor.processAdd(addDocCmd);
}
long end = System.nanoTime();
this.trackerStats.addNodeTime(end - start);
continue NEXT_NODE;
}
}
// Make sure any unindexed or error doc is removed.
if (log.isDebugEnabled()) {
log.debug(".. deleting node " + node.getId());
}
deleteNode(processor, request, node);
SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_NODE);
addToNewDocAndCache(nodeMetaData, doc);
addDocCmd.solrDoc = doc;
// System.out.println("###################### indexing doc:"+doc.toString());
processor.processAdd(addDocCmd);
long end = System.nanoTime();
this.trackerStats.addNodeTime(end - start);
} finally {
// release the lock on the node so other trackers can access the node.
unlock(nodeId);
}
}
// Ends iteration over nodeMetadatas
}
// Ends checking for the existence of updated or unknown node ids
} catch (Exception e) {
log.error("SolrInformationServer problem", e);
// Bulk version failed, so do one at a time.
for (Node node : nodes) {
this.indexNode(node, true);
}
} finally {
if (processor != null) {
processor.finish();
}
if (request != null) {
request.close();
}
}
}
use of org.apache.solr.update.processor.UpdateRequestProcessor in project lucene-solr by apache.
the class BlobHandler method indexMap.
public static void indexMap(SolrQueryRequest req, SolrQueryResponse rsp, Map<String, Object> doc) throws IOException {
SolrInputDocument solrDoc = new SolrInputDocument();
for (Map.Entry<String, Object> e : doc.entrySet()) solrDoc.addField(e.getKey(), e.getValue());
UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessorChain(req.getParams());
try (UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp)) {
AddUpdateCommand cmd = new AddUpdateCommand(req);
cmd.solrDoc = solrDoc;
log.info("Adding doc: " + doc);
processor.processAdd(cmd);
log.info("committing doc: " + doc);
processor.processCommit(new CommitUpdateCommand(req, false));
processor.finish();
}
}
use of org.apache.solr.update.processor.UpdateRequestProcessor in project lucene-solr by apache.
the class UIMAUpdateRequestProcessorTest method testMultiMap.
@Test
public void testMultiMap() {
SolrCore core = h.getCore();
UpdateRequestProcessorChain chained = core.getUpdateProcessingChain(UIMA_MULTI_MAP_CHAIN);
assertNotNull(chained);
UIMAUpdateRequestProcessorFactory factory = (UIMAUpdateRequestProcessorFactory) chained.getProcessors().get(0);
assertNotNull(factory);
UpdateRequestProcessor processor = factory.getInstance(req(), null, null);
assertTrue(processor instanceof UIMAUpdateRequestProcessor);
SolrUIMAConfiguration conf = ((UIMAUpdateRequestProcessor) processor).getConfiguration();
Map<String, Map<String, MapField>> map = conf.getTypesFeaturesFieldsMapping();
Map<String, MapField> subMap = map.get("a-type-which-can-have-multiple-features");
assertEquals(2, subMap.size());
assertEquals("1", subMap.get("A").getFieldName(null));
assertEquals("2", subMap.get("B").getFieldName(null));
}
use of org.apache.solr.update.processor.UpdateRequestProcessor in project lucene-solr by apache.
the class DefaultValueUpdateProcessorTest method processAdd.
/**
* Runs a document through the specified chain, and returns the final
* document used when the chain is completed (NOTE: some chains may
* modify the document in place
*/
SolrInputDocument processAdd(final String chain, final SolrInputDocument docIn) throws IOException {
SolrCore core = h.getCore();
UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
assertNotNull("No Chain named: " + chain, pc);
SolrQueryResponse rsp = new SolrQueryResponse();
SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
try {
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
AddUpdateCommand cmd = new AddUpdateCommand(req);
cmd.solrDoc = docIn;
UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
processor.processAdd(cmd);
return cmd.solrDoc;
} finally {
SolrRequestInfo.clearRequestInfo();
req.close();
}
}
use of org.apache.solr.update.processor.UpdateRequestProcessor in project lucene-solr by apache.
the class ContentStreamHandlerBase method handleRequestBody.
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessorChain(params);
UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
try {
ContentStreamLoader documentLoader = newLoader(req, processor);
Iterable<ContentStream> streams = req.getContentStreams();
if (streams == null) {
if (!RequestHandlerUtils.handleCommit(req, processor, params, false) && !RequestHandlerUtils.handleRollback(req, processor, params, false)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
}
} else {
for (ContentStream stream : streams) {
documentLoader.load(req, rsp, stream, processor);
}
// Perhaps commit from the parameters
RequestHandlerUtils.handleCommit(req, processor, params, false);
RequestHandlerUtils.handleRollback(req, processor, params, false);
}
} finally {
// finish the request
try {
processor.finish();
} finally {
processor.close();
}
}
}
Aggregations