Examples with DocumentProcessor com.yahoo.docproc.DocumentProcessor used on opensource projects

Example 1 with DocumentProcessor

use of com.yahoo.docproc.DocumentProcessor in project vespa by vespa-engine.

the class JDiscContainerDocprocTest method requireThatLaterDocumentProcessingWorks.

@Test
public void requireThatLaterDocumentProcessingWorks() throws Exception {
    try (Application app = new ApplicationBuilder().servicesXml(getXML(CHAIN_NAME, Rot13DocumentProcessor.class.getCanonicalName())).networking(Networking.disable).documentType("music", DOCUMENT).build()) {
        JDisc container = app.getJDisc("container");
        DocumentProcessing docProc = container.documentProcessing();
        DocumentType type = docProc.getDocumentTypes().get("music");
        ChainRegistry<DocumentProcessor> chains = docProc.getChains();
        assertTrue(chains.allComponentsById().containsKey(new ComponentId(CHAIN_NAME)));
        Document doc = new Document(type, "doc:this:is:a:great:album");
        doc.setFieldValue("title", "Great Album!");
        com.yahoo.docproc.Processing processing;
        DocumentProcessor.Progress progress;
        DocumentPut put = new DocumentPut(doc);
        processing = com.yahoo.docproc.Processing.of(put);
        progress = docProc.processOnce(ComponentSpecification.fromString(CHAIN_NAME), processing);
        assertThat(progress, instanceOf(DocumentProcessor.LaterProgress.class));
        assertThat(doc.getFieldValue("title").toString(), equalTo("Great Album!"));
        progress = docProc.processOnce(ComponentSpecification.fromString(CHAIN_NAME), processing);
        assertThat(progress, instanceOf(DocumentProcessor.LaterProgress.class));
        assertThat(doc.getFieldValue("title").toString(), equalTo("Great Album!"));
        progress = docProc.processOnce(ComponentSpecification.fromString(CHAIN_NAME), processing);
        assertThat(progress, sameInstance(DocumentProcessor.Progress.DONE));
        assertThat(doc.getFieldValue("title").toString(), equalTo("Terng Nyohz!"));
    }
}

Example 2 with DocumentProcessor

use of com.yahoo.docproc.DocumentProcessor in project vespa by vespa-engine.

the class SchemaMappingAndAccessesTest method testMappingArrays.

public void testMappingArrays() {
    Document doc = getDoc();
    DocumentProcessor proc = new TestMappingArrayProcessor();
    Map<String, String> fieldMap = new HashMap<>();
    fieldMap.put("label", "labels[0]");
    ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap);
    Processing p = Processing.of(new DocumentPut(mapped));
    proc.process(p);
    assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size());
    assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0));
    assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1));
    fieldMap.clear();
    fieldMap.put("label", "labels[2]");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    try {
        proc.process(p);
        fail("Should not have worked");
    } catch (IllegalArgumentException iae) {
    // ok!
    }
    assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size());
    assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0));
    assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1));
}

Example 3 with DocumentProcessor

use of com.yahoo.docproc.DocumentProcessor in project vespa by vespa-engine.

the class SchemaMappingAndAccessesTest method testMappingStructsInArrays.

public void testMappingStructsInArrays() {
    Document doc = getDoc();
    DocumentProcessor proc = new TestMappingStructInArrayProcessor();
    Map<String, String> fieldMap = new HashMap<>();
    fieldMap.put("name", "listeners[0].firstname");
    ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap);
    Processing p = Processing.of(new DocumentPut(mapped));
    proc.process(p);
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertEquals("and", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString()));
    fieldMap.clear();
    fieldMap.put("name", "listeners[2].firstname");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    try {
        proc.process(p);
        fail("Should not have worked");
    } catch (IllegalArgumentException iae) {
    // ok!
    }
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertEquals("and", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString()));
    // test remove:
    proc = new TestRemovingMappingStructInArrayProcessor();
    fieldMap.clear();
    fieldMap.put("name", "listeners[1].lastname");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    proc.process(p);
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname"));
    fieldMap.clear();
    fieldMap.put("name", "listeners[2].lastname");
    mapped = new ProxyDocument(proc, doc, fieldMap);
    p = Processing.of(new DocumentPut(mapped));
    try {
        proc.process(p);
        fail("Should not have worked");
    } catch (IllegalArgumentException iae) {
    // ok!
    }
    assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size());
    assertEquals("peter", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString()));
    assertEquals("olsen", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString()));
    assertEquals("anders", (((StringFieldValue) ((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString()));
    assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname"));
}

Example 4 with DocumentProcessor

use of com.yahoo.docproc.DocumentProcessor in project vespa by vespa-engine.

the class SchemaMappingAndAccessesTest method testProxyAndSecure.

public void testProxyAndSecure() {
    DocumentProcessor procOK = new TestDPSecure();
    Map<Pair<String, String>, String> fieldMap = new HashMap<>();
    fieldMap.put(new Pair<>("album", "titleMapped"), "title");
    procOK.setFieldMap(fieldMap);
    DocumentPut put = new DocumentPut(getDoc());
    Document proxyDoc = new Call(procOK).configDoc(procOK, put).getDocument();
    procOK.process(Processing.of(new DocumentPut(proxyDoc)));
    assertEquals(proxyDoc.getFieldValue("title").toString(), "MyTitle MyTitle");
}

Example 5 with DocumentProcessor

use of com.yahoo.docproc.DocumentProcessor in project vespa by vespa-engine.

the class DocumentProcessingHandler method convertToCallStack.

private static CallStack convertToCallStack(Chain<DocumentProcessor> chain, Statistics statistics, Metric metric) {
    CallStack stack = new CallStack(chain.getId().stringValue(), statistics, metric);
    for (DocumentProcessor processor : chain.components()) {
        processor.getFieldMap().putAll(DocprocService.schemaMap.chainMap(chain.getId().stringValue(), processor.getId().stringValue()));
        stack.addLast(processor);
    }
    return stack;
}