Examples with BufferSerializer com.yahoo.vespa.objects.BufferSerializer used on opensource projects

Example 6 with BufferSerializer

use of com.yahoo.vespa.objects.BufferSerializer in project vespa by vespa-engine.

the class DocumentTestCase method testDeserialize.

@Test
public void testDeserialize() {
    setUpSertestDocType();
    BufferSerializer buf = new BufferSerializer();
    try {
        new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf()));
        assertTrue(false);
    } catch (Exception e) {
        assertTrue(true);
    }
    buf = BufferSerializer.wrap("Hello world".getBytes());
    try {
        new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf()));
        assertTrue(false);
    } catch (Exception e) {
        assertTrue(true);
    }
}

Example 7 with BufferSerializer

use of com.yahoo.vespa.objects.BufferSerializer in project vespa by vespa-engine.

the class DocumentTestCase method testSerializeDeserializeCompressed.

@Test
public void testSerializeDeserializeCompressed() {
    setUpSertestDocType();
    Document doc = getSertestDocument();
    CompressionConfig noncomp = new CompressionConfig();
    CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
    doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
    doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
    GrowableByteBuffer data = new GrowableByteBuffer();
    doc.serialize(data);
    int size = doc.getSerializedSize();
    doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
    doc.getDataType().getBodyType().setCompressionConfig(noncomp);
    assertEquals(size, data.position());
    data.flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat");
        fos.write(data.array(), 0, data.remaining());
        fos.close();
    } catch (Exception e) {
    }
    Document doc2 = docMan.createDocument(data);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    byte[] docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    byte[] doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
    // Do the same thing, splitting document in two
    BufferSerializer header = new BufferSerializer(new GrowableByteBuffer());
    BufferSerializer body = new BufferSerializer(new GrowableByteBuffer());
    doc.serializeHeader(header);
    doc.serializeBody(body);
    header.getBuf().flip();
    body.getBuf().flip();
    try {
        FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat");
        fos.write(header.getBuf().array(), 0, header.getBuf().remaining());
        fos.close();
        fos = new FileOutputStream("src/test/files/testser-split.body.dat");
        fos.write(body.getBuf().array(), 0, body.getBuf().remaining());
        fos.close();
    } catch (Exception e) {
    }
    DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf());
    doc2 = new Document(deser);
    assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid"));
    assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date"));
    assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from"));
    assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to"));
    assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject"));
    assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body"));
    assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount"));
    assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments"));
    docRawBytes = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer().array();
    doc2RawBytes = ((Raw) doc2.getFieldValue("rawfield")).getByteBuffer().array();
    assertEquals(docRawBytes.length, doc2RawBytes.length);
    for (int i = 0; i < docRawBytes.length; i++) {
        assertEquals(docRawBytes[i], doc2RawBytes[i]);
    }
    assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield"));
    assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield"));
}

Example 8 with BufferSerializer

use of com.yahoo.vespa.objects.BufferSerializer in project vespa by vespa-engine.

the class SerializationHelperTestCase method testGetNullTerminatedString.

public void testGetNullTerminatedString() throws Exception {
    // This is a test.0ab
    byte[] test = { 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x0, 0x61, 0x62 };
    BufferSerializer data = BufferSerializer.wrap(test);
    assertTrue(data.position() == 0);
    Utf8Array thisIsATest = VespaDocumentDeserializer42.parseNullTerminatedString(data.getBuf().getByteBuffer());
    assertTrue(thisIsATest.equals(new Utf8Array(Utf8.toBytes("This is a test."))));
    assertTrue(data.position() == 16);
    // a
    assertTrue(test[16] == 0x61);
    data.position(0);
    assertTrue(data.position() == 0);
    Utf8Array thisIsATestAgain = VespaDocumentDeserializer42.parseNullTerminatedString(data.getBuf().getByteBuffer(), 15);
    assertTrue(thisIsATestAgain.equals(new Utf8Array(Utf8.toBytes("This is a test."))));
    assertTrue(data.position() == 16);
    // a
    assertTrue(test[16] == 0x61);
}

Example 9 with BufferSerializer

use of com.yahoo.vespa.objects.BufferSerializer in project vespa by vespa-engine.

the class VdsVisitor method setVisitorParameters.

private void setVisitorParameters(String searchCluster, Route route) {
    if (query.properties().getString(streamingUserid) != null) {
        params.setDocumentSelection("id.user==" + query.properties().getString(streamingUserid));
    } else if (query.properties().getString(streamingGroupname) != null) {
        params.setDocumentSelection("id.group==\"" + query.properties().getString(streamingGroupname) + "\"");
    } else if (query.properties().getString(streamingSelection) != null) {
        params.setDocumentSelection(query.properties().getString(streamingSelection));
    }
    // Per bucket visitor timeout
    params.setTimeoutMs(query.getTimeout());
    params.setSessionTimeoutMs(query.getTimeout());
    params.setVisitorLibrary("searchvisitor");
    params.setLocalDataHandler(this);
    params.setVisitHeadersOnly(query.properties().getBoolean(streamingHeadersonly));
    if (query.properties().getDouble(streamingFromtimestamp) != null) {
        params.setFromTimestamp(query.properties().getDouble(streamingFromtimestamp).longValue());
    }
    if (query.properties().getDouble(streamingTotimestamp) != null) {
        params.setToTimestamp(query.properties().getDouble(streamingTotimestamp).longValue());
    }
    params.visitInconsistentBuckets(true);
    params.setPriority(DocumentProtocol.Priority.VERY_HIGH);
    if (query.properties().getString(streamingLoadtype) != null) {
        LoadType loadType = visitorSessionFactory.getLoadTypeSet().getNameMap().get(query.properties().getString(streamingLoadtype));
        if (loadType != null) {
            params.setLoadType(loadType);
            params.setPriority(loadType.getPriority());
        }
    }
    if (query.properties().getString(streamingPriority) != null) {
        params.setPriority(DocumentProtocol.getPriorityByName(query.properties().getString(streamingPriority)));
    }
    params.setMaxPending(Integer.MAX_VALUE);
    params.setMaxBucketsPerVisitor(Integer.MAX_VALUE);
    params.setTraceLevel(inferSessionTraceLevel(query));
    String ordering = query.properties().getString(streamingOrdering);
    if (ordering != null) {
        params.setVisitorOrdering(getOrdering(ordering));
        params.setMaxFirstPassHits(query.getOffset() + query.getHits());
        params.setMaxBucketsPerVisitor(1);
        params.setDynamicallyIncreaseMaxBucketsPerVisitor(true);
    }
    String maxbuckets = query.properties().getString(streamingMaxbucketspervisitor);
    if (maxbuckets != null) {
        params.setMaxBucketsPerVisitor(Integer.parseInt(maxbuckets));
    }
    EncodedData ed = new EncodedData();
    encodeQueryData(query, 0, ed);
    params.setLibraryParameter("query", ed.getEncodedData());
    params.setLibraryParameter("querystackcount", String.valueOf(ed.getReturned()));
    params.setLibraryParameter("searchcluster", searchCluster.getBytes());
    if (query.getPresentation().getSummary() != null) {
        params.setLibraryParameter("summaryclass", query.getPresentation().getSummary());
    } else {
        params.setLibraryParameter("summaryclass", "default");
    }
    params.setLibraryParameter("summarycount", String.valueOf(query.getOffset() + query.getHits()));
    params.setLibraryParameter("rankprofile", query.getRanking().getProfile());
    params.setLibraryParameter("allowslimedocsums", "true");
    params.setLibraryParameter("queryflags", String.valueOf(getQueryFlags(query)));
    ByteBuffer buf = ByteBuffer.allocate(1024);
    if (query.getRanking().getLocation() != null) {
        buf.clear();
        query.getRanking().getLocation().encode(buf);
        buf.flip();
        byte[] af = new byte[buf.remaining()];
        buf.get(af);
        params.setLibraryParameter("location", af);
    }
    if (query.hasEncodableProperties()) {
        encodeQueryData(query, 1, ed);
        params.setLibraryParameter("rankproperties", ed.getEncodedData());
    }
    List<Grouping> groupingList = GroupingExecutor.getGroupingList(query);
    if (groupingList.size() > 0) {
        BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer());
        gbuf.putInt(null, groupingList.size());
        for (Grouping g : groupingList) {
            g.serialize(gbuf);
        }
        gbuf.flip();
        byte[] blob = gbuf.getBytes(null, gbuf.getBuf().limit());
        params.setLibraryParameter("aggregation", blob);
    }
    if (query.getRanking().getSorting() != null) {
        encodeQueryData(query, 3, ed);
        params.setLibraryParameter("sort", ed.getEncodedData());
    }
    params.setRoute(route);
}

Example 10 with BufferSerializer

use of com.yahoo.vespa.objects.BufferSerializer in project vespa by vespa-engine.

the class VdsVisitorTestCase method createSR.

private SearchResult createSR(String docId, double rank) {
    BufferSerializer serializer = new BufferSerializer();
    // total hits
    serializer.putInt(null, 2);
    // hit count
    serializer.putInt(null, 1);
    serializer.put(null, docId);
    serializer.putDouble(null, rank);
    // sort blob count
    serializer.putInt(null, 0);
    // aggregator count
    serializer.putInt(null, 0);
    // grouping count
    serializer.putInt(null, 0);
    serializer.getBuf().flip();
    return new SearchResult(serializer);
}