Examples with GrowableByteBuffer com.yahoo.io.GrowableByteBuffer used on opensource projects

Example 26 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class AnnotationTestCase method serializeAndAssert.

private void serializeAndAssert(Annotation annotation) {
    GrowableByteBuffer buffer = new GrowableByteBuffer(1024);
    DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer);
    serializer.write(annotation);
    buffer.flip();
    DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer);
    Annotation annotation2 = new Annotation();
    deserializer.read(annotation2);
    assertEquals(annotation, annotation2);
    assertNotSame(annotation, annotation2);
}

Example 27 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class MixedBinaryFormat method decodeCells.

private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) {
    List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList());
    TensorType sparseType = MixedTensor.createPartialType(sparseDimensions);
    long denseSubspaceSize = builder.denseSubspaceSize();
    int numBlocks = 1;
    if (sparseDimensions.size() > 0) {
        numBlocks = buffer.getInt1_4Bytes();
    }
    double[] denseSubspace = new double[(int) denseSubspaceSize];
    for (int i = 0; i < numBlocks; ++i) {
        TensorAddress.Builder sparseAddress = new TensorAddress.Builder(sparseType);
        for (TensorType.Dimension sparseDimension : sparseDimensions) {
            sparseAddress.add(sparseDimension.name(), buffer.getUtf8String());
        }
        for (long denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) {
            denseSubspace[(int) denseOffset] = buffer.getDouble();
        }
        builder.block(sparseAddress.build(), denseSubspace);
    }
}

Example 28 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class VdsVisitor method setVisitorParameters.

private void setVisitorParameters(String searchCluster, Route route) {
    if (query.properties().getString(streamingUserid) != null) {
        params.setDocumentSelection("id.user==" + query.properties().getString(streamingUserid));
    } else if (query.properties().getString(streamingGroupname) != null) {
        params.setDocumentSelection("id.group==\"" + query.properties().getString(streamingGroupname) + "\"");
    } else if (query.properties().getString(streamingSelection) != null) {
        params.setDocumentSelection(query.properties().getString(streamingSelection));
    }
    // Per bucket visitor timeout
    params.setTimeoutMs(query.getTimeout());
    params.setSessionTimeoutMs(query.getTimeout());
    params.setVisitorLibrary("searchvisitor");
    params.setLocalDataHandler(this);
    params.setVisitHeadersOnly(query.properties().getBoolean(streamingHeadersonly));
    if (query.properties().getDouble(streamingFromtimestamp) != null) {
        params.setFromTimestamp(query.properties().getDouble(streamingFromtimestamp).longValue());
    }
    if (query.properties().getDouble(streamingTotimestamp) != null) {
        params.setToTimestamp(query.properties().getDouble(streamingTotimestamp).longValue());
    }
    params.visitInconsistentBuckets(true);
    params.setPriority(DocumentProtocol.Priority.VERY_HIGH);
    if (query.properties().getString(streamingLoadtype) != null) {
        LoadType loadType = visitorSessionFactory.getLoadTypeSet().getNameMap().get(query.properties().getString(streamingLoadtype));
        if (loadType != null) {
            params.setLoadType(loadType);
            params.setPriority(loadType.getPriority());
        }
    }
    if (query.properties().getString(streamingPriority) != null) {
        params.setPriority(DocumentProtocol.getPriorityByName(query.properties().getString(streamingPriority)));
    }
    params.setMaxPending(Integer.MAX_VALUE);
    params.setMaxBucketsPerVisitor(Integer.MAX_VALUE);
    params.setTraceLevel(inferSessionTraceLevel(query));
    String ordering = query.properties().getString(streamingOrdering);
    if (ordering != null) {
        params.setVisitorOrdering(getOrdering(ordering));
        params.setMaxFirstPassHits(query.getOffset() + query.getHits());
        params.setMaxBucketsPerVisitor(1);
        params.setDynamicallyIncreaseMaxBucketsPerVisitor(true);
    }
    String maxbuckets = query.properties().getString(streamingMaxbucketspervisitor);
    if (maxbuckets != null) {
        params.setMaxBucketsPerVisitor(Integer.parseInt(maxbuckets));
    }
    EncodedData ed = new EncodedData();
    encodeQueryData(query, 0, ed);
    params.setLibraryParameter("query", ed.getEncodedData());
    params.setLibraryParameter("querystackcount", String.valueOf(ed.getReturned()));
    params.setLibraryParameter("searchcluster", searchCluster.getBytes());
    if (query.getPresentation().getSummary() != null) {
        params.setLibraryParameter("summaryclass", query.getPresentation().getSummary());
    } else {
        params.setLibraryParameter("summaryclass", "default");
    }
    params.setLibraryParameter("summarycount", String.valueOf(query.getOffset() + query.getHits()));
    params.setLibraryParameter("rankprofile", query.getRanking().getProfile());
    params.setLibraryParameter("allowslimedocsums", "true");
    params.setLibraryParameter("queryflags", String.valueOf(getQueryFlags(query)));
    ByteBuffer buf = ByteBuffer.allocate(1024);
    if (query.getRanking().getLocation() != null) {
        buf.clear();
        query.getRanking().getLocation().encode(buf);
        buf.flip();
        byte[] af = new byte[buf.remaining()];
        buf.get(af);
        params.setLibraryParameter("location", af);
    }
    if (query.hasEncodableProperties()) {
        encodeQueryData(query, 1, ed);
        params.setLibraryParameter("rankproperties", ed.getEncodedData());
    }
    List<Grouping> groupingList = GroupingExecutor.getGroupingList(query);
    if (groupingList.size() > 0) {
        BufferSerializer gbuf = new BufferSerializer(new GrowableByteBuffer());
        gbuf.putInt(null, groupingList.size());
        for (Grouping g : groupingList) {
            g.serialize(gbuf);
        }
        gbuf.flip();
        byte[] blob = gbuf.getBytes(null, gbuf.getBuf().limit());
        params.setLibraryParameter("aggregation", blob);
    }
    if (query.getRanking().getSorting() != null) {
        encodeQueryData(query, 3, ed);
        params.setLibraryParameter("sort", ed.getEncodedData());
    }
    params.setRoute(route);
}

Example 29 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class SerializeTestCase method testSimple.

@Test
public void testSimple() {
    SomeIdClass s = new SomeIdClass();
    BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer());
    s.serialize(buf);
    buf.flip();
    s.deserialize(buf);
}

Example 30 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class SerializeTestCase method testIdNotFound.

@Test
public void testIdNotFound() {
    BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer());
    buf.putInt(null, 717273);
    buf.flip();
    boolean caught = false;
    try {
        Identifiable nsi = Identifiable.create(buf);
    } catch (IllegalArgumentException e) {
        // System.out.println(e);
        caught = true;
    }
    assertThat(caught, is(true));
}