use of com.yahoo.docproc.Processing in project vespa by vespa-engine.
the class DocprocMessageProcessor method processDocumentOperations.
public void processDocumentOperations(List<DocumentOperation> documentOperations, Message m) throws Exception {
Processing processing = Processing.createProcessingFromDocumentOperations(docproc.getName(), documentOperations, new CallStack(docproc.getCallStack()));
processing.setServiceName(docproc.getName());
processing.setDocprocServiceRegistry(docprocServiceRegistry);
processing.setVariable("route", m.getRoute());
processing.setVariable("timeout", m.getTimeRemaining());
DocumentProcessor.Progress progress = docproc.getExecutor().process(processing);
while (DocumentProcessor.Progress.LATER.equals(progress)) {
Thread.sleep(50);
progress = docproc.getExecutor().process(processing);
}
if (progress == DocumentProcessor.Progress.FAILED || progress == DocumentProcessor.Progress.PERMANENT_FAILURE) {
throw new RuntimeException("Processing of " + documentOperations + " failed: " + progress + ".");
}
m.setRoute((Route) processing.getVariable("route"));
m.setTimeRemaining((Long) processing.getVariable("timeout"));
}
use of com.yahoo.docproc.Processing in project vespa by vespa-engine.
the class SchemaMappingAndAccessesTest method testMappingArrays.
public void testMappingArrays() {
Document doc = getDoc();
DocumentProcessor proc = new TestMappingArrayProcessor();
Map<String, String> fieldMap = new HashMap<>();
fieldMap.put("label", "labels[0]");
ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap);
Processing p = Processing.of(new DocumentPut(mapped));
proc.process(p);
assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size());
assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0));
assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1));
fieldMap.clear();
fieldMap.put("label", "labels[2]");
mapped = new ProxyDocument(proc, doc, fieldMap);
p = Processing.of(new DocumentPut(mapped));
try {
proc.process(p);
fail("Should not have worked");
} catch (IllegalArgumentException iae) {
// ok!
}
assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size());
assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0));
assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1));
}
use of com.yahoo.docproc.Processing in project vespa by vespa-engine.
the class SchemaMappingAndAccessesTest method testMappingStructsInArrays.
public void testMappingStructsInArrays() {
Document doc = getDoc();
DocumentProcessor proc = new TestMappingStructInArrayProcessor();
Map<String, String> fieldMap = new HashMap<>();
fieldMap.put("name", "listeners[0].firstname");
ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap);
Processing p = Processing.of(new DocumentPut(mapped));
proc.process(p);
assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
assertEquals("and", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString()));
fieldMap.clear();
fieldMap.put("name", "listeners[2].firstname");
mapped = new ProxyDocument(proc, doc, fieldMap);
p = Processing.of(new DocumentPut(mapped));
try {
proc.process(p);
fail("Should not have worked");
} catch (IllegalArgumentException iae) {
// ok!
}
assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
assertEquals("and", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString()));
// test remove:
proc = new TestRemovingMappingStructInArrayProcessor();
fieldMap.clear();
fieldMap.put("name", "listeners[1].lastname");
mapped = new ProxyDocument(proc, doc, fieldMap);
p = Processing.of(new DocumentPut(mapped));
proc.process(p);
assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname"));
fieldMap.clear();
fieldMap.put("name", "listeners[2].lastname");
mapped = new ProxyDocument(proc, doc, fieldMap);
p = Processing.of(new DocumentPut(mapped));
try {
proc.process(p);
fail("Should not have worked");
} catch (IllegalArgumentException iae) {
// ok!
}
assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname"));
}
use of com.yahoo.docproc.Processing in project vespa by vespa-engine.
the class IndexingProcessorTestCase method process.
private DocumentOperation process(DocumentOperation input) {
Processing proc = new Processing();
proc.getDocumentOperations().add(input);
indexer.process(proc);
List<DocumentOperation> lst = proc.getDocumentOperations();
assertEquals(1, lst.size());
return lst.get(0);
}
use of com.yahoo.docproc.Processing in project vespa by vespa-engine.
the class SplitterJoinerTestCase method testSplitJoin.
@Test
public void testSplitJoin() {
ConfigGetter<SplitterJoinerDocumentProcessorConfig> getter = new ConfigGetter<>(SplitterJoinerDocumentProcessorConfig.class);
ConfigGetter<DocumentmanagerConfig> docManGetter = new ConfigGetter<>(DocumentmanagerConfig.class);
SplitterJoinerDocumentProcessorConfig cfg = getter.getConfig("file:src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg");
DocumentmanagerConfig docManCfg = docManGetter.getConfig("file:src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg");
SplitterDocumentProcessor splitter = new SplitterDocumentProcessor(cfg, docManCfg);
DocumentTypeManager manager = splitter.manager;
/**
** Create documents: ***
*/
Document inner1 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:one");
inner1.setFieldValue("name", new StringFieldValue("Donald Duck"));
inner1.setFieldValue("content", new StringFieldValue("Lives in Duckburg"));
Document inner2 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:two");
inner2.setFieldValue("name", new StringFieldValue("Uncle Scrooge"));
inner2.setFieldValue("content", new StringFieldValue("Lives in Duckburg, too."));
Array<Document> innerArray = (Array<Document>) manager.getDocumentType("outerdoc").getField("innerdocuments").getDataType().createFieldValue();
innerArray.add(inner1);
innerArray.add(inner2);
Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one");
outer.setFieldValue("innerdocuments", innerArray);
/**
** End create documents ***
*/
Processing p = Processing.of(new DocumentPut(outer));
splitter.process(p);
assertEquals(2, p.getDocumentOperations().size());
assertThat(((DocumentPut) (p.getDocumentOperations().get(0))).getDocument(), sameInstance(inner1));
assertThat(((DocumentPut) (p.getDocumentOperations().get(1))).getDocument(), sameInstance(inner2));
assertThat(((DocumentPut) (p.getVariable(cfg.contextFieldName()))).getDocument(), sameInstance(outer));
assertThat(outer.getFieldValue("innerdocuments"), sameInstance(innerArray));
assertTrue(innerArray.isEmpty());
JoinerDocumentProcessor joiner = new JoinerDocumentProcessor(cfg, docManCfg);
joiner.process(p);
assertThat(p.getDocumentOperations().size(), equalTo(1));
assertThat(((DocumentPut) p.getDocumentOperations().get(0)).getDocument(), sameInstance(outer));
assertThat(p.getVariable(cfg.contextFieldName()), nullValue());
assertThat(outer.getFieldValue("innerdocuments"), sameInstance(innerArray));
assertThat(innerArray.size(), equalTo(2));
assertThat(innerArray.get(0), sameInstance(inner1));
assertThat(innerArray.get(1), sameInstance(inner2));
}
Aggregations