Search in sources :

Example 1 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class DocprocMessageProcessor method processDocumentOperations.

public void processDocumentOperations(List<DocumentOperation> documentOperations, Message m) throws Exception {
    Processing processing = Processing.createProcessingFromDocumentOperations(docproc.getName(), documentOperations, new CallStack(docproc.getCallStack()));
    processing.setServiceName(docproc.getName());
    processing.setDocprocServiceRegistry(docprocServiceRegistry);
    processing.setVariable("route", m.getRoute());
    processing.setVariable("timeout", m.getTimeRemaining());
    DocumentProcessor.Progress progress = docproc.getExecutor().process(processing);
    while (DocumentProcessor.Progress.LATER.equals(progress)) {
        Thread.sleep(50);
        progress = docproc.getExecutor().process(processing);
    }
    if (progress == DocumentProcessor.Progress.FAILED || progress == DocumentProcessor.Progress.PERMANENT_FAILURE) {
        throw new RuntimeException("Processing of " + documentOperations + " failed: " + progress + ".");
    }
    m.setRoute((Route) processing.getVariable("route"));
    m.setTimeRemaining((Long) processing.getVariable("timeout"));
}
Also used : CallStack(com.yahoo.docproc.CallStack) DocumentProcessor(com.yahoo.docproc.DocumentProcessor) Processing(com.yahoo.docproc.Processing)

Example 2 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class SchemaMappingAndAccessesTest method testMappingArrays.

public void testMappingArrays() {
    Document doc = getDoc();
    DocumentProcessor proc = new TestMappingArrayProcessor();
    Map<String, String> fieldMap = new HashMap<>();
    fieldMap.put("label", "labels[0]");
    ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap);
    Processing p = Processing.of(new DocumentPut(mapped));
    proc.process(p);
    assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size());
    assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0));
    assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1));
    fieldMap.clear();
    fieldMap.put("label", "labels[2]");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    try {
        proc.process(p);
        fail("Should not have worked");
    } catch (IllegalArgumentException iae) {
    // ok!
    }
    assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size());
    assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0));
    assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1));
}
Also used : DocumentProcessor(com.yahoo.docproc.DocumentProcessor) HashMap(java.util.HashMap) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) DocumentPut(com.yahoo.document.DocumentPut) Document(com.yahoo.document.Document) Processing(com.yahoo.docproc.Processing)

Example 3 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class SchemaMappingAndAccessesTest method testMappingStructsInArrays.

public void testMappingStructsInArrays() {
    Document doc = getDoc();
    DocumentProcessor proc = new TestMappingStructInArrayProcessor();
    Map<String, String> fieldMap = new HashMap<>();
    fieldMap.put("name", "listeners[0].firstname");
    ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap);
    Processing p = Processing.of(new DocumentPut(mapped));
    proc.process(p);
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertEquals("and", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString()));
    fieldMap.clear();
    fieldMap.put("name", "listeners[2].firstname");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    try {
        proc.process(p);
        fail("Should not have worked");
    } catch (IllegalArgumentException iae) {
    // ok!
    }
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertEquals("and", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString()));
    // test remove:
    proc = new TestRemovingMappingStructInArrayProcessor();
    fieldMap.clear();
    fieldMap.put("name", "listeners[1].lastname");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    proc.process(p);
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname"));
    fieldMap.clear();
    fieldMap.put("name", "listeners[2].lastname");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    try {
        proc.process(p);
        fail("Should not have worked");
    } catch (IllegalArgumentException iae) {
    // ok!
    }
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname"));
}
Also used : DocumentProcessor(com.yahoo.docproc.DocumentProcessor) HashMap(java.util.HashMap) DocumentPut(com.yahoo.document.DocumentPut) Document(com.yahoo.document.Document) Processing(com.yahoo.docproc.Processing) Struct(com.yahoo.document.datatypes.Struct) Array(com.yahoo.document.datatypes.Array) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue)

Example 4 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class IndexingProcessorTestCase method process.

private DocumentOperation process(DocumentOperation input) {
    Processing proc = new Processing();
    proc.getDocumentOperations().add(input);
    indexer.process(proc);
    List<DocumentOperation> lst = proc.getDocumentOperations();
    assertEquals(1, lst.size());
    return lst.get(0);
}
Also used : DocumentOperation(com.yahoo.document.DocumentOperation) Processing(com.yahoo.docproc.Processing)

Example 5 with Processing

use of com.yahoo.docproc.Processing in project vespa by vespa-engine.

the class SplitterJoinerTestCase method testSplitJoin.

@Test
public void testSplitJoin() {
    ConfigGetter<SplitterJoinerDocumentProcessorConfig> getter = new ConfigGetter<>(SplitterJoinerDocumentProcessorConfig.class);
    ConfigGetter<DocumentmanagerConfig> docManGetter = new ConfigGetter<>(DocumentmanagerConfig.class);
    SplitterJoinerDocumentProcessorConfig cfg = getter.getConfig("file:src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg");
    DocumentmanagerConfig docManCfg = docManGetter.getConfig("file:src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg");
    SplitterDocumentProcessor splitter = new SplitterDocumentProcessor(cfg, docManCfg);
    DocumentTypeManager manager = splitter.manager;
    /**
     ** Create documents: ***
     */
    Document inner1 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:one");
    inner1.setFieldValue("name", new StringFieldValue("Donald Duck"));
    inner1.setFieldValue("content", new StringFieldValue("Lives in Duckburg"));
    Document inner2 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:two");
    inner2.setFieldValue("name", new StringFieldValue("Uncle Scrooge"));
    inner2.setFieldValue("content", new StringFieldValue("Lives in Duckburg, too."));
    Array<Document> innerArray = (Array<Document>) manager.getDocumentType("outerdoc").getField("innerdocuments").getDataType().createFieldValue();
    innerArray.add(inner1);
    innerArray.add(inner2);
    Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one");
    outer.setFieldValue("innerdocuments", innerArray);
    /**
     ** End create documents ***
     */
    Processing p = Processing.of(new DocumentPut(outer));
    splitter.process(p);
    assertEquals(2, p.getDocumentOperations().size());
    assertThat(((DocumentPut) (p.getDocumentOperations().get(0))).getDocument(), sameInstance(inner1));
    assertThat(((DocumentPut) (p.getDocumentOperations().get(1))).getDocument(), sameInstance(inner2));
    assertThat(((DocumentPut) (p.getVariable(cfg.contextFieldName()))).getDocument(), sameInstance(outer));
    assertThat(outer.getFieldValue("innerdocuments"), sameInstance(innerArray));
    assertTrue(innerArray.isEmpty());
    JoinerDocumentProcessor joiner = new JoinerDocumentProcessor(cfg, docManCfg);
    joiner.process(p);
    assertThat(p.getDocumentOperations().size(), equalTo(1));
    assertThat(((DocumentPut) p.getDocumentOperations().get(0)).getDocument(), sameInstance(outer));
    assertThat(p.getVariable(cfg.contextFieldName()), nullValue());
    assertThat(outer.getFieldValue("innerdocuments"), sameInstance(innerArray));
    assertThat(innerArray.size(), equalTo(2));
    assertThat(innerArray.get(0), sameInstance(inner1));
    assertThat(innerArray.get(1), sameInstance(inner2));
}
Also used : ConfigGetter(com.yahoo.config.subscription.ConfigGetter) DocumentPut(com.yahoo.document.DocumentPut) SplitterJoinerDocumentProcessorConfig(com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig) Document(com.yahoo.document.Document) Processing(com.yahoo.docproc.Processing) Array(com.yahoo.document.datatypes.Array) DocumentmanagerConfig(com.yahoo.document.config.DocumentmanagerConfig) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) DocumentTypeManager(com.yahoo.document.DocumentTypeManager) Test(org.junit.Test)

Aggregations

Processing (com.yahoo.docproc.Processing)10 DocumentProcessor (com.yahoo.docproc.DocumentProcessor)4 Document (com.yahoo.document.Document)3 DocumentPut (com.yahoo.document.DocumentPut)3 StringFieldValue (com.yahoo.document.datatypes.StringFieldValue)3 Test (org.junit.Test)3 CallStack (com.yahoo.docproc.CallStack)2 DocumentOperation (com.yahoo.document.DocumentOperation)2 Array (com.yahoo.document.datatypes.Array)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 Tuple2 (com.yahoo.collections.Tuple2)1 SplitterJoinerDocumentProcessorConfig (com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig)1 ConfigGetter (com.yahoo.config.subscription.ConfigGetter)1 Call (com.yahoo.docproc.Call)1 HandledProcessingException (com.yahoo.docproc.HandledProcessingException)1 DocumentRemove (com.yahoo.document.DocumentRemove)1 DocumentType (com.yahoo.document.DocumentType)1 DocumentTypeManager (com.yahoo.document.DocumentTypeManager)1 DocumentUpdate (com.yahoo.document.DocumentUpdate)1