Examples with Processing com.yahoo.docproc.Processing used on opensource projects

Example 6 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class MbusRequestContext method processingDone.

@Override
public void processingDone(List<Processing> processings) {
    List<DocumentMessage> messages = new ArrayList<>();
    if (messageFactory != null) {
        for (Processing processing : processings) {
            for (DocumentOperation documentOperation : processing.getDocumentOperations()) {
                messages.add(messageFactory.fromDocumentOperation(processing, documentOperation));
            }
        }
    }
    if (log.isLoggable(LogLevel.DEBUG)) {
        log.log(LogLevel.DEBUG, "Forwarding " + messages.size() + " messages from " + processings.size() + " processings.");
    }
    if (messages.isEmpty()) {
        dispatchResponse(Response.Status.OK);
        return;
    }
    long inputSequenceId = requestMsg.getSequenceId();
    ResponseMerger responseHandler = new ResponseMerger(requestMsg, messages.size(), this);
    for (Message message : messages) {
        // See comment for internalNoThrottledSource.
        dispatchRequest(message, (inputSequenceId == message.getSequenceId()) ? getUri().getPath() : "/" + internalNoThrottledSource, responseHandler);
    }
}

Example 7 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class ApplicationTest method empty_container.

@Test
public void empty_container() throws Exception {
    try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container())))) {
        try {
            app.process(new DocumentRemove(null));
            fail("expected exception");
        } catch (Exception ignore) {
        // no op
        }
        try {
            app.process(new Processing());
            fail("expected exception");
        } catch (Exception ignore) {
        // no op
        }
        try {
            app.search(new Query("?foo"));
            fail("expected exception");
        } catch (Exception ignore) {
        // no op
        }
    }
}

Example 8 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class IndexingProcessorTestCase method requireThatEmptyDocumentUpdateOutputDoesNotThrow.

@Test
public void requireThatEmptyDocumentUpdateOutputDoesNotThrow() {
    DocumentType inputType = indexer.getDocumentTypeManager().getDocumentType("music");
    DocumentUpdate input = new DocumentUpdate(inputType, "doc:scheme:");
    Processing proc = new Processing();
    proc.getDocumentOperations().add(input);
    indexer.process(proc);
    assertEquals(0, proc.getDocumentOperations().size());
}

Example 9 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class DocumentProcessingTask method process.

/**
 * Processes a single Processing, and fails the message if this processing fails.
 *
 * @param executor the DocprocService to use for processing
 */
private DocumentProcessor.Progress process(DocprocExecutor executor) {
    Iterator<Processing> iterator = processings.iterator();
    List<Tuple2<DocumentProcessor.Progress, Processing>> later = new ArrayList<>();
    while (iterator.hasNext()) {
        Processing processing = iterator.next();
        iterator.remove();
        if (requestContext.hasExpired()) {
            DocumentProcessor.Progress progress = DocumentProcessor.Progress.FAILED;
            final String location;
            if (processing != null) {
                final CallStack callStack = processing.callStack();
                if (callStack != null) {
                    final Call lastPopped = callStack.getLastPopped();
                    if (lastPopped != null) {
                        location = lastPopped.toString();
                    } else {
                        location = "empty call stack or no processors popped";
                    }
                } else {
                    location = "no call stack";
                }
            } else {
                location = "no processing instance";
            }
            String errorMsg = processing + " failed, " + location;
            log.log(Level.FINE, "Time is up for '" + errorMsg + "'.");
            requestContext.processingFailed(RequestContext.ErrorCode.ERROR_PROCESSING_FAILURE, "Time is up.");
            return progress;
        }
        DocumentProcessor.Progress progress = DocumentProcessor.Progress.FAILED;
        try {
            progress = executor.process(processing);
        } catch (Exception e) {
            logProcessingFailure(processing, e);
            requestContext.processingFailed(e);
            return progress;
        }
        if (DocumentProcessor.Progress.LATER.equals(progress)) {
            later.add(new Tuple2<>(progress, processing));
        } else if (DocumentProcessor.Progress.DONE.equals(progress)) {
            processingsDone.add(processing);
        } else if (DocumentProcessor.Progress.FAILED.equals(progress)) {
            logProcessingFailure(processing, null);
            requestContext.processingFailed(RequestContext.ErrorCode.ERROR_PROCESSING_FAILURE, progress.getReason().orElse("Document processing failed."));
            return progress;
        } else if (DocumentProcessor.Progress.PERMANENT_FAILURE.equals(progress)) {
            logProcessingFailure(processing, null);
            requestContext.processingFailed(RequestContext.ErrorCode.ERROR_PROCESSING_FAILURE, progress.getReason().orElse("Document processing failed."));
            return progress;
        }
    }
    if (!later.isEmpty()) {
        // Outdated comment:
        // "if this was a multioperationmessage and more than one of the processings returned LATER,
        // return the one with the lowest timeout:"
        // As multioperation is removed this can probably be simplified?
        DocumentProcessor.LaterProgress shortestDelay = (DocumentProcessor.LaterProgress) later.get(0).first;
        for (Tuple2<DocumentProcessor.Progress, Processing> tuple : later) {
            // re-add the LATER one to processings
            processings.add(tuple.second);
            // check to see if this one had a lower timeout than the previous one:
            if (((DocumentProcessor.LaterProgress) tuple.first).getDelay() < shortestDelay.getDelay()) {
                shortestDelay = (DocumentProcessor.LaterProgress) tuple.first;
            }
        }
        return shortestDelay;
    } else {
        requestContext.processingDone(processingsDone);
        return DocumentProcessor.Progress.DONE;
    }
}

Example 10 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class ProcessingFactory method createProcessing.

private Processing createProcessing(DocumentOperation documentOperation, Message message) {
    Processing processing = new Processing();
    processing.addDocumentOperation(documentOperation);
    processing.setServiceName(serviceName);
    processing.setDocprocServiceRegistry(docprocServiceComponentRegistry);
    processing.setVariable("route", message.getRoute());
    processing.setVariable("timeout", message.getTimeRemaining());
    return processing;
}