Examples with Raw com.yahoo.document.datatypes.Raw used on opensource projects

Example 1 with Raw

use of com.yahoo.document.datatypes.Raw in project vespa by vespa-engine.

the class DocumentFieldTemplate method hit.

@Override
public void hit(Context context, Writer writer) throws IOException {
    DocumentHit hit = (DocumentHit) context.get("hit");
    Document doc = hit.getDocument();
    // Assume field existence has been checked before we ever get here.
    // Also assume that relevant encoding/content type is set
    // appropriately according to the request and the field's content
    // type, as this is immutable in the template set.
    FieldValue value = doc.getFieldValue(field);
    if (field.getDataType() == DataType.RAW) {
        ByteWriter bw = (ByteWriter) writer;
        bw.append(((Raw) value).getByteBuffer().array());
    } else {
        writer.write(XML.xmlEscape(value.toString(), false));
    }
}

Example 2 with Raw

use of com.yahoo.document.datatypes.Raw in project vespa by vespa-engine.

the class GetSearcherTestCase method testDocumentFieldRawContent.

@Test
public void testDocumentFieldRawContent() throws Exception {
    byte[] contentBytes = new byte[] { 0, -128, 127 };
    Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456"));
    doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes)));
    GetDocumentReply[] replies = new GetDocumentReply[] { new GetDocumentReply(doc1) };
    Chain<Searcher> searchChain = createSearcherChain(replies);
    Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:123:456&field=foo"));
    assertNull(result.hits().getErrorHit());
    assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType());
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    SearchRendererAdaptor.callRender(stream, result);
    stream.flush();
    byte[] resultBytes = stream.toByteArray();
    assertEquals(contentBytes.length, resultBytes.length);
    for (int i = 0; i < resultBytes.length; ++i) {
        assertEquals(contentBytes[i], resultBytes[i]);
    }
}

Example 3 with Raw

use of com.yahoo.document.datatypes.Raw in project vespa by vespa-engine.

the class DocumentTestCase method testSerializeDeserialize.

@Test
public void testSerializeDeserialize() {
    setUpSertestDocType();
    Document doc = getSertestDocument();
    GrowableByteBuffer data = new GrowableByteBuffer();
    doc.serialize(data);
    int size = doc.getSerializedSize();
    assertEquals(size, data.position());
    data.flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
        fos.write(data.array(), 0, data.remaining());
        fos.close();
    } catch (Exception e) {
    }
    Document doc2 = docMan.createDocument(data);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    byte[] docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    byte[] doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    // Do the same thing, splitting document in two
    DocumentSerializer header = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true);
    DocumentSerializer body = DocumentSerializerFactory.create42(new GrowableByteBuffer());
    doc.serializeHeader(header);
    doc.serializeBody(body);
    header.getBuf().flip();
    body.getBuf().flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
        fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
        fos.close();
        fos = new FileOutputStream("src/test/files/testser-split.body.dat");
        fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
        fos.close();
    } catch (Exception e) {
    }
    DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
    doc2 = new Document(deser);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    Document docInDoc = (Document) doc.getFieldValue("docindoc");
    assert (docInDoc != null);
    assertEquals(new StringFieldValue("ball"), docInDoc.getFieldValue("tull"));
}

Example 4 with Raw

use of com.yahoo.document.datatypes.Raw in project vespa by vespa-engine.

the class DocumentTestCase method testCppDocSplit.

@Test
public void testCppDocSplit() throws IOException {
    docMan = setUpCppDocType();
    byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
    byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat");
    DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData), GrowableByteBuffer.wrap(bodyData));
    Document doc = new Document(header);
    assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
    assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
    assertEquals(new FloatFieldValue((float) -9.23), doc.getFieldValue("floatfield"));
    assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
    assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
    assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
    assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
    // NOTE: The value really is unsigned 254, which becomes signed -2:
    assertEquals(new ByteFieldValue((byte) -2), doc.getFieldValue("bytefield"));
    ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
    assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
    Document docindoc = (Document) doc.getFieldValue("docfield");
    assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
    assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
    WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
    assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
    assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
}

Example 5 with Raw

use of com.yahoo.document.datatypes.Raw in project vespa by vespa-engine.

the class DocumentTestCase method testSerializeDeserializeCompressed.

@Test
public void testSerializeDeserializeCompressed() {
    setUpSertestDocType();
    Document doc = getSertestDocument();
    CompressionConfig noncomp = new CompressionConfig();
    CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
    doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
    doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
    GrowableByteBuffer data = new GrowableByteBuffer();
    doc.serialize(data);
    int size = doc.getSerializedSize();
    doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
    doc.getDataType().getBodyType().setCompressionConfig(noncomp);
    assertEquals(size, data.position());
    data.flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
        fos.write(data.array(), 0, data.remaining());
        fos.close();
    } catch (Exception e) {
    }
    Document doc2 = docMan.createDocument(data);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    byte[] docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    byte[] doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    // Do the same thing, splitting document in two
    BufferSerializer header = new BufferSerializer(new GrowableByteBuffer());
    BufferSerializer body = new BufferSerializer(new GrowableByteBuffer());
    doc.serializeHeader(header);
    doc.serializeBody(body);
    header.getBuf().flip();
    body.getBuf().flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
        fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
        fos.close();
        fos = new FileOutputStream("src/test/files/testser-split.body.dat");
        fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
        fos.close();
    } catch (Exception e) {
    }
    DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
    doc2 = new Document(deser);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
}